Gradio demo works with Confz now.

.gitignore

@@ -2,3 +2,4 @@
 .vscode
 __pycache__
 .env
+config.yml.debug

config.yml

@@ -0,0 +1,37 @@
+llms:
+  anthropic/claude-3-haiku:
+    type: ChatOpenAI
+    temperature: 0.1
+    model_name: "anthropic/claude-3-haiku:beta"
+    openai_api_key: ""
+    openai_api_base: "https://openrouter.ai/api/v1"
+    max_tokens: 8192
+    verbose: true
+  anthropic/claude-3-sonnet:
+    type: ChatOpenAI
+    temperature: 0.1
+    model_name: "anthropic/claude-3-sonnet:beta"
+    openai_api_key: ""
+    openai_api_base: "https://openrouter.ai/api/v1"
+    max_tokens: 8192
+    verbose: true
+  anthropic/deepseek-chat:
+    type: ChatOpenAI
+    temperature: 0.1
+    model_name: "deepseek/deepseek-chat"
+    openai_api_key: ""
+    openai_api_base: "https://openrouter.ai/api/v1"
+    max_tokens: 8192
+    verbose: true
+  groq/llama3-70b-8192:
+    type: ChatOpenAI
+    temperature: 0.1
+    model_name: "llama3-70b-8192"
+    openai_api_key: ""
+    openai_api_base: "https://api.groq.com/openai/v1"
+    max_tokens: 8192
+    verbose: true
+
+examples_path: "demo/examples"
+server_name: 0.0.0.0
+server_port: 7870

demo/config.py

@@ -0,0 +1,16 @@
+# config.py
+from confz import BaseConfig
+from pydantic import BaseModel, Extra
+from typing import Optional
+
+class LLMConfig(BaseModel):
+    type: str
+
+    class Config:
+        extra = Extra.allow
+
+class MetaPromptConfig(BaseConfig):
+    llms: Optional[dict[str, LLMConfig]]
+    examples_path: Optional[str]
+    server_name: Optional[str] = '127.0.0.1'
+    server_port: Optional[int] = 7878

demo/examples/log.csv

@@ -188,10 +188,12 @@ What is the meaning of life?,"[
   * Data types and formats of all JSON fields
   * Top layer sections
 * Acceptable differences:
-  * Differences in field values
+  * Different personas or prompts
+  * Different numbers of personas
   * Extra or missing spaces
   * Extra or missing line breaks at the beginning or end of the output
-  * JSON wrapped in backquotes"
+* Unacceptable:
+  * Showing the personas in Expected Output in System Message"
 "<?php
 $username = $_POST['username'];
 $password = $_POST['password'];

demo/gradio_meta_prompt.py

@@ -1,31 +1,40 @@
 import gradio as gr
+from confz import BaseConfig, CLArgSource, EnvSource, FileSource
 from meta_prompt import MetaPromptGraph, AgentState
 from langchain_openai import ChatOpenAI
+from config import MetaPromptConfig
 
-# Initialize the MetaPromptGraph with the required LLMs
-MODEL_NAME = "anthropic/claude-3.5-sonnet:haiku"
-# MODEL_NAME = "meta-llama/llama-3-70b-instruct"
-# MODEL_NAME = "deepseek/deepseek-chat"
-# MODEL_NAME = "google/gemma-2-9b-it"
-# MODEL_NAME = "recursal/eagle-7b"
-# MODEL_NAME = "meta-llama/llama-3-8b-instruct"
-llm = ChatOpenAI(model_name=MODEL_NAME)
-meta_prompt_graph = MetaPromptGraph(llms=llm)
-
-def process_message(user_message, expected_output, acceptance_criteria, recursion_limit: int=25):
+class LLMModelFactory:
+    def __init__(self):
+        pass
+
+    def create(self, model_type: str, **kwargs):
+        model_class = globals()[model_type]
+        return model_class(**kwargs)
+    
+llm_model_factory = LLMModelFactory()
+
+def process_message(user_message, expected_output, acceptance_criteria, initial_system_message,
+                    recursion_limit: int, model_name: str):
     # Create the input state
     input_state = AgentState(
         user_message=user_message,
         expected_output=expected_output,
-        acceptance_criteria=acceptance_criteria
+        acceptance_criteria=acceptance_criteria,
+        system_message=initial_system_message
     )
     
     # Get the output state from MetaPromptGraph
+    type = config.llms[model_name].type
+    args = config.llms[model_name].model_dump(exclude={'type'})
+    llm = llm_model_factory.create(type, **args)
+    meta_prompt_graph = MetaPromptGraph(llms=llm)
     output_state = meta_prompt_graph(input_state, recursion_limit=recursion_limit)
     
     # Validate the output state
     system_message = ''
     output = ''
+    analysis = ''
 
     if 'best_system_message' in output_state and output_state['best_system_message'] is not None:
         system_message = output_state['best_system_message']
@@ -37,22 +46,58 @@ def process_message(user_message, expected_output, acceptance_criteria, recursio
     else:
         output = "Error: The output state does not contain a valid 'best_output'"
 
-    return system_message, output
+    if 'analysis' in output_state and output_state['analysis'] is not None:
+        analysis = output_state['analysis']
+    else:
+        analysis = "Error: The output state does not contain a valid 'analysis'"
+
+    return system_message, output, analysis
+
+class FileConfig(BaseConfig):
+    config_file: str = 'config.yml'  # default path
+
+pre_config_sources = [
+    EnvSource(prefix='METAPROMPT_', allow_all=True),
+    CLArgSource()
+]
+pre_config = FileConfig(config_sources=pre_config_sources)
+
+config_sources = [
+    FileSource(file=pre_config.config_file, optional=True),
+    EnvSource(prefix='METAPROMPT_', allow_all=True),
+    CLArgSource()
+]
+
+config = MetaPromptConfig(config_sources=config_sources)
 
 # Create the Gradio interface
 iface = gr.Interface(
     fn=process_message,
     inputs=[
-        gr.Textbox(label="User Message"),
-        gr.Textbox(label="Expected Output"),
-        gr.Textbox(label="Acceptance Criteria"),
-        gr.Number(label="Recursion Limit", value=25, precision=0, minimum=1, maximum=100, step=1)
+        gr.Textbox(label="User Message", show_copy_button=True),
+        gr.Textbox(label="Expected Output", show_copy_button=True),
+        gr.Textbox(label="Acceptance Criteria", show_copy_button=True),
+    ],
+    outputs=[
+        gr.Textbox(label="System Message", show_copy_button=True),
+        gr.Textbox(label="Output", show_copy_button=True),
+        gr.Textbox(label="Analysis", show_copy_button=True)
+    ],
+    additional_inputs=[
+        gr.Textbox(label="Initial System Message", show_copy_button=True, value=""),
+        gr.Number(label="Recursion Limit", value=25,
+                  precision=0, minimum=1, maximum=100, step=1),
+        gr.Dropdown(
+            label="Model Name",
+            choices=config.llms.keys(),
+            value=list(config.llms.keys())[0],
+        )
     ],
-    outputs=[gr.Textbox(label="System Message"), gr.Textbox(label="Output")],
+    # stop_btn = gr.Button("Stop", variant="stop", visible=True),
     title="MetaPromptGraph Chat Interface",
     description="A chat interface for MetaPromptGraph to process user inputs and generate system messages.",
-    examples="demo/examples"
+    examples=config.examples_path
 )
 
 # Launch the Gradio app
-iface.launch()
+iface.launch(server_name=config.server_name, server_port=config.server_port)

requirements.txt

@@ -1,5 +1,5 @@
 aiofiles==23.2.1
-aiohttp==3.8.5
+aiohttp==3.9.5
 aiosignal==1.3.1
 altair==5.1.1
 annotated-types==0.5.0
@@ -11,6 +11,7 @@ certifi==2023.7.22
 charset-normalizer==3.2.0
 click==8.1.7
 comm==0.2.2
+confz==2.0.1
 contourpy==1.1.1
 cycler==0.11.0
 dataclasses-json==0.6.0
@@ -25,16 +26,17 @@ filelock==3.12.4
 fonttools==4.42.1
 frozenlist==1.4.0
 fsspec==2023.9.2
-gradio==3.44.4
-gradio_client==0.5.1
+gradio==4.37.2
+gradio_client==1.0.2
 greenlet==2.0.2
 h11==0.14.0
 httpcore==0.18.0
 httpx==0.25.0
-huggingface-hub==0.17.2
+huggingface-hub==0.23.4
 idna==3.4
 importlib-resources==6.1.0
 ipykernel==6.29.4
+ipython==8.26.0
 jedi==0.19.1
 Jinja2==3.1.2
 joblib==1.3.2
@@ -51,10 +53,12 @@ langchain-openai==0.1.13
 langchain-text-splitters==0.2.2
 langgraph==0.1.4
 langsmith==0.1.82
+markdown-it-py==3.0.0
 MarkupSafe==2.1.3
 marshmallow==3.20.1
 matplotlib==3.8.0
 matplotlib-inline==0.1.7
+mdurl==0.1.2
 multidict==6.0.4
 mypy-extensions==1.0.0
 nest-asyncio==1.6.0
@@ -78,17 +82,21 @@ pydub==0.25.1
 Pygments==2.18.0
 pyparsing==3.1.1
 python-dateutil==2.9.0.post0
-python-multipart==0.0.6
+python-dotenv==1.0.1
+python-multipart==0.0.9
 pytz==2023.3.post1
 PyYAML==6.0.1
 pyzmq==26.0.3
 referencing==0.30.2
 regex==2024.5.15
 requests==2.31.0
+rich==13.7.1
 rpds-py==0.10.3
+ruff==0.5.0
 scikit-learn==1.3.1
 scipy==1.11.3
 semantic-version==2.10.0
+shellingham==1.5.4
 six==1.16.0
 sniffio==1.3.0
 SQLAlchemy==2.0.21
@@ -97,10 +105,13 @@ starlette==0.27.0
 tenacity==8.2.3
 threadpoolctl==3.2.0
 tiktoken==0.7.0
+toml==0.10.2
+tomlkit==0.12.0
 toolz==0.12.0
 tornado==6.4.1
 tqdm==4.66.1
 traitlets==5.14.3
+typer==0.12.3
 typing-inspect==0.9.0
 typing_extensions==4.12.2
 tzdata==2023.3

src/meta_prompt/meta_prompt.py

@@ -164,23 +164,9 @@ If both outputs are equally similar to the expected output, output the following
         ]),
         NODE_PROMPT_ANALYZER: ChatPromptTemplate.from_messages([
             ("system", """
-You are a text comparing program. You compare the following output texts and provide a
-detailed analysis according to `Acceptance Criteria`. Then you decide whether `Actual Output`
-is acceptable.
-
-# Expected Output
-
-```
-{expected_output}
-```
-
-# Actual Output
-
-```
-{output}
-```
-
-----
+You are a text comparing program. You compare the following output texts,
+analysis the System Message and provide a detailed analysis according to
+`Acceptance Criteria`. Then you decide whether `Actual Output` is acceptable.
 
 Provide your analysis in the following format:
 
@@ -200,6 +186,25 @@ Provide your analysis in the following format:
 ```
 {acceptance_criteria}
 ```
+"""),
+            ("human", """
+# System Message
+
+```
+{system_message}
+```
+
+# Expected Output
+
+```
+{expected_output}
+```
+
+# Actual Output
+
+```
+{output}
+```
 """)
         ]),
         NODE_PROMPT_SUGGESTER: ChatPromptTemplate.from_messages([
@@ -281,42 +286,34 @@ Analysis:
         self.prompt_templates: Dict[str, ChatPromptTemplate] = self.DEFAULT_PROMPT_TEMPLATES.copy()
         self.prompt_templates.update(prompts)
 
-        # create workflow
-        self.workflow = StateGraph(AgentState)
-
-        self.workflow.add_node(self.NODE_PROMPT_INITIAL_DEVELOPER,
-                               lambda x: self._prompt_node(
-                                   self.NODE_PROMPT_INITIAL_DEVELOPER,
-                                   "system_message",
-                                   x))
-        self.workflow.add_node(self.NODE_PROMPT_DEVELOPER,
-                               lambda x: self._prompt_node(
-                                   self.NODE_PROMPT_DEVELOPER,
-                                   "system_message",
-                                   x))
-        self.workflow.add_node(self.NODE_PROMPT_EXECUTOR,
-                               lambda x: self._prompt_node(
-                                   self.NODE_PROMPT_EXECUTOR,
-                                   "output",
-                                   x))
-        self.workflow.add_node(self.NODE_OUTPUT_HISTORY_ANALYZER,
-                               lambda x: self._output_history_analyzer(x))
-        self.workflow.add_node(self.NODE_PROMPT_ANALYZER,
-                               lambda x: self._prompt_analyzer(x))
-        self.workflow.add_node(self.NODE_PROMPT_SUGGESTER,
-                               lambda x: self._prompt_node(
-                                   self.NODE_PROMPT_SUGGESTER,
-                                   "suggestions",
-                                   x))
-
-        self.workflow.set_entry_point(self.NODE_PROMPT_INITIAL_DEVELOPER)
-
-        self.workflow.add_edge(self.NODE_PROMPT_INITIAL_DEVELOPER, self.NODE_PROMPT_EXECUTOR)
-        self.workflow.add_edge(self.NODE_PROMPT_DEVELOPER, self.NODE_PROMPT_EXECUTOR)
-        self.workflow.add_edge(self.NODE_PROMPT_EXECUTOR, self.NODE_OUTPUT_HISTORY_ANALYZER)
-        self.workflow.add_edge(self.NODE_PROMPT_SUGGESTER, self.NODE_PROMPT_DEVELOPER)
-
-        self.workflow.add_conditional_edges(
+    def _create_workflow(self, including_initial_developer: bool = True) -> StateGraph:
+        workflow = StateGraph(AgentState)
+            
+        workflow.add_node(self.NODE_PROMPT_DEVELOPER,
+                          lambda x: self._prompt_node(
+                              self.NODE_PROMPT_DEVELOPER,
+                              "system_message",
+                              x))
+        workflow.add_node(self.NODE_PROMPT_EXECUTOR,
+                          lambda x: self._prompt_node(
+                              self.NODE_PROMPT_EXECUTOR,
+                              "output",
+                              x))
+        workflow.add_node(self.NODE_OUTPUT_HISTORY_ANALYZER,
+                          lambda x: self._output_history_analyzer(x))
+        workflow.add_node(self.NODE_PROMPT_ANALYZER,
+                          lambda x: self._prompt_analyzer(x))
+        workflow.add_node(self.NODE_PROMPT_SUGGESTER,
+                          lambda x: self._prompt_node(
+                              self.NODE_PROMPT_SUGGESTER,
+                              "suggestions",
+                              x))
+
+        workflow.add_edge(self.NODE_PROMPT_DEVELOPER, self.NODE_PROMPT_EXECUTOR)
+        workflow.add_edge(self.NODE_PROMPT_EXECUTOR, self.NODE_OUTPUT_HISTORY_ANALYZER)
+        workflow.add_edge(self.NODE_PROMPT_SUGGESTER, self.NODE_PROMPT_DEVELOPER)
+
+        workflow.add_conditional_edges(
             self.NODE_OUTPUT_HISTORY_ANALYZER,
             lambda x: self._should_exit_on_max_age(x),
             {
@@ -326,7 +323,7 @@ Analysis:
             }
         )
 
-        self.workflow.add_conditional_edges(
+        workflow.add_conditional_edges(
             self.NODE_PROMPT_ANALYZER,
             lambda x: self._should_exit_on_acceptable_output(x),
             {
@@ -335,9 +332,24 @@ Analysis:
             }
         )
 
+        if including_initial_developer:
+            workflow.add_node(self.NODE_PROMPT_INITIAL_DEVELOPER,
+                            lambda x: self._prompt_node(
+                                self.NODE_PROMPT_INITIAL_DEVELOPER,
+                                "system_message",
+                                x))
+            workflow.add_edge(self.NODE_PROMPT_INITIAL_DEVELOPER, self.NODE_PROMPT_EXECUTOR)
+            workflow.set_entry_point(self.NODE_PROMPT_INITIAL_DEVELOPER)
+        else:
+            workflow.set_entry_point(self.NODE_PROMPT_EXECUTOR)
+
+        return workflow
+
     def __call__(self, state: AgentState, recursion_limit: int = 25) -> AgentState:
+        workflow = self._create_workflow(including_initial_developer=(state.system_message is None or state.system_message == ""))
+
         memory = MemorySaver()
-        graph = self.workflow.compile(checkpointer=memory)
+        graph = workflow.compile(checkpointer=memory)
         config = {"configurable": {"thread_id": "1"}, "recursion_limit": recursion_limit}
 
         try: