Refactor code and update system message formatting guidelines.

app/examples/log.csv

@@ -329,3 +329,4 @@ Therefore, the area of each triangle is 15 cm^2, 6 cm^2, 8 cm^2, 60 cm^2, and 31
 * Acceptable differences:
   * Different code examples
   * Minor text differences","As a Code Tutor Assistant, you are designed to handle advanced-level queries that require expertise in writing and explaining code snippets."
+"If it takes 8 bits to make a byte, how many bits are there in a kilobyte?","There are 8,192 bits in a kilobyte. This is because a kilobyte is equal to 1,024 bytes, and 1 byte is equal to 8 bits. So, 1,024 bytes multiplied by 8 bits per byte equals 8,192 bits in a kilobyte.",Exactly format and style match. Consistent semantic. Highly similar text length.,

app/gradio_meta_prompt.py

@@ -148,7 +148,13 @@ def evaluate_system_message(system_message, user_message, simple_model, executor
         ("human", "{user_message}")
     ])
     messages = template.format_messages(system_message=system_message, user_message=user_message)
-    output = llm.invoke(messages)
+    try:
+        output = llm.invoke(messages)
+    except Exception as e:
+        if isinstance(e, gr.Error):
+            raise e
+        else:
+            raise gr.Error(f"Error: {e}")   
 
     if hasattr(output, 'content'):
         return output.content
@@ -160,7 +166,6 @@ def process_message(user_message, expected_output, acceptance_criteria,
                     initial_system_message, recursion_limit: int,
                     max_output_age: int,
                     llms: Union[BaseLanguageModel, Dict[str, BaseLanguageModel]]):
-    # Create the input state
     input_state = AgentState(
         user_message=user_message,
         expected_output=expected_output,
@@ -169,49 +174,35 @@ def process_message(user_message, expected_output, acceptance_criteria,
         max_output_age=max_output_age
     )
 
-    # Get the output state from MetaPromptGraph
     log_stream = io.StringIO()
-    log_handler = None
-    logger = None
-    if config.verbose:
-        log_handler = logging.StreamHandler(log_stream)
-        logger = logging.getLogger(MetaPromptGraph.__name__)
+    logger = logging.getLogger(MetaPromptGraph.__name__) if config.verbose else None
+    log_handler = logging.StreamHandler(log_stream) if logger else None
+    if log_handler:
         log_handler.setFormatter(jsonlogger.JsonFormatter(
             '%(asctime)s %(name)s %(levelname)s %(message)s'))
         logger.addHandler(log_handler)
 
-    meta_prompt_graph = MetaPromptGraph(
-        llms=llms, verbose=config.verbose, logger=logger)
-    output_state = meta_prompt_graph(input_state, recursion_limit=recursion_limit)
+    meta_prompt_graph = MetaPromptGraph(llms=llms, verbose=config.verbose, logger=logger)
+    try:
+        output_state = meta_prompt_graph(input_state, recursion_limit=recursion_limit)
+    except Exception as e:
+        if isinstance(e, gr.Error):
+            raise e
+        else:
+            raise gr.Error(f"Error: {e}")           
 
-    if config.verbose:
+    if log_handler:
         log_handler.close()
         log_output = log_stream.getvalue()
     else:
         log_output = None
 
-    # Validate the output state
-    system_message = ''
-    output = ''
-    analysis = ''
-
-    if 'best_system_message' in output_state and output_state['best_system_message'] is not None:
-        system_message = output_state['best_system_message']
-    else:
-        system_message = "Error: The output state does not contain a valid 'best_system_message'"
-
-    if 'best_output' in output_state and output_state['best_output'] is not None:
-        output = output_state["best_output"]
-    else:
-        output = "Error: The output state does not contain a valid 'best_output'"
+    system_message = output_state.get('best_system_message', "Error: The output state does not contain a valid 'best_system_message'")
+    output = output_state.get('best_output', "Error: The output state does not contain a valid 'best_output'")
+    analysis = output_state.get('analysis', "Error: The output state does not contain a valid 'analysis'")
 
-    if 'analysis' in output_state and output_state['analysis'] is not None:
-        analysis = output_state['analysis']
-    else:
-        analysis = "Error: The output state does not contain a valid 'analysis'"
+    return (system_message, output, analysis, chat_log_2_chatbot_list(log_output))
 
-    return (system_message, output, analysis,
-            chat_log_2_chatbot_list(log_output))
 
 
 def process_message_with_single_llm(user_message, expected_output, acceptance_criteria, initial_system_message,

meta_prompt/consts.py

@@ -28,7 +28,8 @@ You are an expert prompt engineer tasked with creating system messages for AI as
 2. Ensure the system message can handle similar user messages.
 3. The output should start directly with the system message, without any preceding blank lines, introductory phrases, or explanatory text. Do not include extra lines at the beginning or end of the output.
 4. Expected Output text should not appear in System Message as an example. But it's OK to use some similar text as an example instead.
-5. Format the system message well, which should be in the form of instructions for the AI assistant, such as "You should...". Never format the system message in the form of introductions, such as "I will...".
+5. In the System Message, do not use `Expected Output` to refer to the example you want to illustrate. Instead, directly describe the specific features you need.
+6. Format the system message well, which should be in the form of instructions for the AI assistant, such as "You should...". Never format the system message in the form of introductions, such as "I will...".
 
 ## Output
 
@@ -59,8 +60,9 @@ You are an expert prompt engineer tasked with updating system messages for AI as
 4. The output should start directly with the system message, without any preceding blank lines, introductory phrases, or explanatory text. Do not include extra lines at the beginning or end of the output.
 5. Avoiding the behavior should be explicitly requested (e.g. `Don't ...`) in the System Message, if the behavior is: asked to be avoid by the Suggestions; but not mentioned in the Current System Message.
 6. Expected Output text should not appear in System Message as an example. But it's OK to use some similar text as an example instead.
-7. Remove the Expected Output text or text highly similar to Expected Output from System Message, if it's present.
-8. Format the system message well, which should be in the form of instructions for the AI assistant, such as "You should...". Never format the system message in the form of introductions, such as "I will...".
+7. In the System Message, do not use `Expected Output` to refer to the example you want to illustrate. Instead, directly describe the specific features you need.
+8. Remove the Expected Output text or text highly similar to Expected Output from System Message, if it's present.
+9. Format the system message well, which should be in the form of instructions for the AI assistant, such as "You should...". Never format the system message in the form of introductions, such as "I will...".
 
 ## Output
 
@@ -91,12 +93,12 @@ Provide only the updated System Message, adhering to the above guidelines.
         ("human", "{user_message}")
     ]),
     NODE_OUTPUT_HISTORY_ANALYZER: ChatPromptTemplate.from_messages([
-        ("system", """You are a text comparing program. You read the Acceptance Criteria, compare the compare the exptected output with two different outputs, and decide which one is more consistent with the expected output. When comparing the outputs, ignore the differences which are acceptable or ignorable according to the Acceptance Criteria.
+        ("system", """You are a text comparing program. You read the Acceptance Criteria, compare the compare the Expected Output with two different outputs, and decide which one is more consistent with the Expected Output. When comparing the outputs, ignore the differences which are acceptable or ignorable according to the Acceptance Criteria.
 
 You output the following analysis according to the Acceptance Criteria:
 
 * Your analysis in a Markdown list.
-* Indicates an output ID that is more consistent with the expected output, in the following format:
+* Indicates an output ID that is more consistent with the Expected Output, in the following format:
 
 ```
 # Analysis
@@ -106,7 +108,7 @@ You output the following analysis according to the Acceptance Criteria:
 # Preferred Output ID: [ID]
 ```
 
-If both outputs are equally similar to the expected output, output the following:
+If both outputs are equally similar to the Expected Output, output the following:
 
 ```
 # Analysis
@@ -191,6 +193,7 @@ Provide your analysis in the following format:
 * Provide your suggestions in a Markdown list, nothing else. Output only the suggestions related with Unacceptable Differences.
 * Start every suggestion with `The System Message should ...`.
 * Figue out the contexts of the System Message that conflict with the suggestions, and suggest modification or deletion.
+* Do not simply describe the output as being the same/similar/different from the Expected Output, such as `the output should not use a different format and style compared to the Expected Output` or `the output should match the expected output exactly`; instead, describe the expected characteristics specifically and suggest a detailed example.
 * Avoiding the behavior should be explicitly requested (e.g. `The System Message should explicitly state that the output shoud not ...`) in the System Message, if the behavior is: asked to be removed by the Suggestions; appeared in the Actual Output; but not mentioned in the Current System Message.
 * Expected Output text should not appear in System Message as an example. But it's OK to use some similar but distinct text as an example instead.
 * Ask to remove the Expected Output text or text highly similar to Expected Output from System Message, if it's present.

tests/meta_prompt_graph_test.py

@@ -4,11 +4,11 @@ import logging
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
-# Assuming the necessary imports are made for the classes and functions used in meta_prompt_graph.py
-from meta_prompt import AgentState, MetaPromptGraph
-
 from langchain_openai import ChatOpenAI
 
+# Assuming the necessary imports are made for the classes and functions used in meta_prompt_graph.py
+from meta_prompt import *
+
 class TestMetaPromptGraph(unittest.TestCase):
     def setUp(self):
         # logging.basicConfig(level=logging.DEBUG)
@@ -16,7 +16,7 @@ class TestMetaPromptGraph(unittest.TestCase):
 
     def test_prompt_node(self):
         llms = {
-            MetaPromptGraph.NODE_PROMPT_INITIAL_DEVELOPER: MagicMock(
+            NODE_PROMPT_INITIAL_DEVELOPER: MagicMock(
                 invoke=MagicMock(return_value=MagicMock(content="Mocked response content"))
             )
         }
@@ -29,7 +29,7 @@ class TestMetaPromptGraph(unittest.TestCase):
 
         # Invoke the _prompt_node method with the mock node, target attribute, and state
         updated_state = graph._prompt_node(
-            MetaPromptGraph.NODE_PROMPT_INITIAL_DEVELOPER, "output", state
+            NODE_PROMPT_INITIAL_DEVELOPER, "output", state
         )
 
         # Assertions
@@ -70,7 +70,7 @@ class TestMetaPromptGraph(unittest.TestCase):
 
     def test_prompt_analyzer_accept(self):
         llms = {
-            MetaPromptGraph.NODE_PROMPT_ANALYZER: MagicMock(
+            NODE_PROMPT_ANALYZER: MagicMock(
                 invoke=lambda prompt: MagicMock(content="Accept: Yes"))
         }
         meta_prompt_graph = MetaPromptGraph(llms)
@@ -78,6 +78,11 @@ class TestMetaPromptGraph(unittest.TestCase):
         updated_state = meta_prompt_graph._prompt_analyzer(state)
         assert updated_state.accepted == True
 
+    def test_get_node_names(self):
+        graph = MetaPromptGraph()
+        node_names = graph.get_node_names()
+        self.assertEqual(node_names, META_PROMPT_NODES)
+
     def test_workflow_execution(self):
         # MODEL_NAME = "anthropic/claude-3.5-sonnet:beta"
         # MODEL_NAME = "meta-llama/llama-3-70b-instruct"
@@ -120,12 +125,12 @@ class TestMetaPromptGraph(unittest.TestCase):
         executor_llm = ChatOpenAI(model_name="meta-llama/llama-3-8b-instruct", temperature=0.01)
 
         llms = {
-            MetaPromptGraph.NODE_PROMPT_INITIAL_DEVELOPER: optimizer_llm,
-            MetaPromptGraph.NODE_PROMPT_DEVELOPER: optimizer_llm,
-            MetaPromptGraph.NODE_PROMPT_EXECUTOR: executor_llm,
-            MetaPromptGraph.NODE_OUTPUT_HISTORY_ANALYZER: optimizer_llm,
-            MetaPromptGraph.NODE_PROMPT_ANALYZER: optimizer_llm,
-            MetaPromptGraph.NODE_PROMPT_SUGGESTER: optimizer_llm
+            NODE_PROMPT_INITIAL_DEVELOPER: optimizer_llm,
+            NODE_PROMPT_DEVELOPER: optimizer_llm,
+            NODE_PROMPT_EXECUTOR: executor_llm,
+            NODE_OUTPUT_HISTORY_ANALYZER: optimizer_llm,
+            NODE_PROMPT_ANALYZER: optimizer_llm,
+            NODE_PROMPT_SUGGESTER: optimizer_llm
         }
 
         meta_prompt_graph = MetaPromptGraph(llms=llms)