Merge branch 'lib' into hf-main

.github/workflows/docker-image.yml

@@ -0,0 +1,41 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  # build:
+
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v3
+  #   - name: Build the Docker image
+  #     run: docker build . --file Dockerfile --tag yaleh/meta-prompt:$(date +%s)
+
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and Publish Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: |
+            yaleh/meta-prompt:${{ github.sha }}
+            ${{ github.ref == 'refs/heads/main' && 'yaleh/meta-prompt:latest' || '' }}
+    

.gitignore

@@ -0,0 +1,7 @@
+.venv
+.vscode
+__pycache__
+.env
+config.yml.debug
+debug.yml
+dist

Dockerfile

@@ -0,0 +1,20 @@
+# Use an official Python runtime as the base image
+FROM python:3.10
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy all files from the current directory to the working directory in the container
+COPY config.yml poetry.lock pyproject.toml /app/
+COPY app /app/app/
+COPY meta_prompt /app/meta_prompt/
+
+RUN pip install --no-cache-dir -U poetry
+RUN poetry config virtualenvs.create false
+RUN poetry install --with=dev
+
+# Expose the port (if necessary)
+EXPOSE 7860
+
+# Run the script when the container launches
+CMD python app/gradio_meta_prompt.py

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Yale Huang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,10 +1,264 @@
+# Meta Prompt Demo
+
+This project is a demonstration of the concept of Meta Prompt, which involves generating a language model prompt using another language model. The demo showcases how a language model can be used to generate high-quality prompts for another language model.
+
+[![Meta Prompt](https://img.youtube.com/vi/eNFUq2AjKCk/0.jpg)](https://www.youtube.com/watch?v=eNFUq2AjKCk)
+
+## Overview
+
+The demo utilizes OpenAI's language models and provides a user interface for interacting with the chatbot. It allows users to input prompts, execute model calls, compare outputs, and optimize prompts based on desired criteria.
+
+**(2023/10/15)** A new working mode called `Other User Prompts` has been added. In the prompt optimization process, similar compatible prompts to the original user prompt are referenced to significantly reduce iteration cycles.
+
+**(2024/06/30)** The `langgraph_meta_prompt.ipynb` file, committed on 06/30/2024, introduces a sophisticated framework for generating and refining system messages for AI assistants. Powered by LangGraph, this notebook can generate high-quality prompts with much more models, including `claude-3.5-sonnet:beta`, `llama-3-70b-instruct`, and quite some with 70B+ parameters. The notebook also introduced a new approach to converge the system messages automatically.
+
+## Try it out!
+
+| Name                   | Colab Notebook                                                                                                                       |
+|------------------------|----------------------------------------------------------------------------------------------------------------------------|
+| Meta Prompt            | [![Open meta_prompt.ipynb In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/yaleh/meta-prompt/blob/main/meta_prompt.ipynb)            |
+| LangGraph Meta Prompt  | [![Open langgraph_meta_prompt.ipynb In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/yaleh/meta-prompt/blob/main/langgraph_meta_prompt.ipynb)  |
+
+## Installation
+
+To use this demo, please follow these steps:
+
+1. Clone the repository: `git clone https://github.com/yaleh/meta-prompt.git`
+2. Change into the project directory: `cd meta-prompt`
+3. Install the required dependencies: `pip install -r requirements.txt`
+
+Please note that you need to have Python and pip installed on your system.
+
+## Usage
+
+To run the demo, execute the following command:
+
+```
+python meta_prompt.py --api_key YOUR_API_KEY
+```
+
+Replace `YOUR_API_KEY` with your OpenAI API key. Other optional parameters can be specified as well, such as proxy settings, model name, API base URL, maximum message length, sharing option, and advanced mode. Please refer to the command-line argument options in the script for more details.
+
+Once the demo is running, you can interact with the chatbot through the user interface provided. Enter prompts, execute model calls, compare outputs, and explore the functionality of the Meta Prompt concept.
+
+To perform the demo on the web, follow these steps:
+
+1. Fill in the user prompt in the "Testing User Prompt" section with a prompt suitable for training/testing.
+2. Fill in the expected output in the "Expected Output" section to specify the desired response from the model.
+3. Set the "Optimize Iterations" parameter. It is recommended to start with 1 iteration and gradually increase it later.
+4. Click on "Optimize Prompts" or "Single Step Optimize" to optimize (generate) the prompt.
+5. After generating the "New System Prompt," click "Run New" to validate it using the "New System Prompt" and "Testing User Prompt."
+6. If the "New Output" is better than the "Current Output," click "Accept New Prompt" to copy the "New System Prompt" and "New Output" to the "Current System Prompt" and "Current Output," respectively, as a basis for further optimization.
+7. Adjust the "Optimize Iterations" and optimize again.
+
+Usually, simple questions (such as arithmetic operations) require around 3 iterations of optimization, while complex problems may require more than 10 iterations.
+
+### Settings
+
+It is recommended to use GPT-4 as the Generating LLM Model for running the meta prompt. GPT-3.5 may not reliably generate the expected results for most questions.
+
+You can use either GPT-4 or GPT-3.5 as the Testing LLM Model, similar to when using GPT/ChatGPT in regular scenarios.
+
+If you have access to ChatGPT and want to save costs on GPT-4 API usage, you can also manually execute the meta-prompt by clicking "Merge Meta System Prompt." This will generate a complete prompt, including the meta-prompt and the current example, that can be used with ChatGPT. However, note that if further iterations are required, you need to manually copy the newly generated system prompt to the Current System Prompt and click "Run Current" to update the Current Output.
+
+## Running Docker Image
+
+To perform the demo using Docker, make sure you have Docker installed on your system, and then follow these steps:
+
+1. Pull the Meta Prompt Docker image by running the following command:
+
+```
+docker pull yaleh/meta-prompt
+```
+
+2. Run the Docker container with the following command:
+
+```
+docker run -d --name meta-prompt-container -p 7860:7860 -e API_KEY=YOUR_API_KEY -e OTHER_ARGS="--advanced_mode" -e OPENAI_API_BASE=https://openai.lrfz.com/v1 yaleh/meta-prompt
+```
+
+Replace `YOUR_API_KEY` with your OpenAI API key. You can modify other environment variables if needed.
+3. You can now access the Meta Prompt demo by opening your web browser and visiting `http://localhost:7860`.
+
+To stop and remove the Meta Prompt container, run the following commands:
+
+```
+docker stop meta-prompt-container
+docker rm meta-prompt-container
+```
+
+Usually, simple questions (such as arithmetic operations) require around 3 iterations of optimization, while complex problems may require more than 10 iterations.
+
+## Examples
+
+### Arithmetic
+
+#### Testing User Prompt
+
+```
+(2+8)*3
+```
+
+#### Expected Output
+
+```
+(2+8)*3
+= 10*3 
+= 30
+```
+
+#### Prompt After 4 Interations
+
+```
+ROLE
+
+You are a math tutor.
+
+TASK
+
+Your task is to solve the mathematical expression provided by the user and provide a concise, step-by-step solution. Each step should only include the calculation and the result, without any additional explanations or step labels.
+
+REQUIREMENTS AND RESTRICTIONS
+
+* The solution should be provided in standard mathematical notation.
+* The format of the mathematical expressions should be consistent with the user's input.
+* The symbols used in the mathematical expressions should be consistent with the user's input.
+* No spaces should be included around the mathematical operators.
+* Avoid unnecessary explanations or verbosity.
+* Do not include any additional information or explanations beyond the direct calculation steps.
+* Do not include a final solution statement.
+
+{user_message}
+```
+
+### GDP
+
+#### Testing User Prompt
+
+```
+Here is the GDP data in billions of US dollars (USD) for these years:
+
+Germany:
+
+2015: $3,368.29 billion
+2016: $3,467.79 billion
+2017: $3,677.83 billion
+2018: $3,946.00 billion
+2019: $3,845.03 billion
+France:
+
+2015: $2,423.47 billion
+2016: $2,465.12 billion
+2017: $2,582.49 billion
+2018: $2,787.86 billion
+2019: $2,715.52 billion
+United Kingdom:
+
+2015: $2,860.58 billion
+2016: $2,650.90 billion
+2017: $2,622.43 billion
+2018: $2,828.87 billion
+2019: $2,829.21 billion
+Italy:
+
+2015: $1,815.72 billion
+2016: $1,852.50 billion
+2017: $1,937.80 billion
+2018: $2,073.90 billion
+2019: $1,988.14 billion
+Spain:
+
+2015: $1,199.74 billion
+2016: $1,235.95 billion
+2017: $1,313.13 billion
+2018: $1,426.19 billion
+2019: $1,430.38 billion
+
+```
+
+#### Expected Output
+
+```
+Year,Germany,France,United Kingdom,Italy,Spain
+2016-2015,2.96%,1.71%,-7.35%,2.02%,3.04%
+2017-2016,5.08%,4.78%,-1.07%,4.61%,6.23%
+2018-2017,7.48%,7.99%,7.89%,7.10%,8.58%
+2019-2018,-2.56%,-2.59%,0.01%,-4.11%,0.30%
+```
+
+#### Other User Prompts
+
+```
+Here is the GDP data in billions of US dollars (USD) for these years:
+
+1. China:
+   - 2010: $6,101.18 billion
+   - 2011: $7,572.80 billion
+   - 2012: $8,560.59 billion
+   - 2013: $9,607.23 billion
+   - 2014: $10,482.65 billion
+
+2. India:
+   - 2010: $1,675.62 billion
+   - 2011: $1,823.05 billion
+   - 2012: $1,827.64 billion
+   - 2013: $1,856.72 billion
+   - 2014: $2,046.88 billion
+
+3. Japan:
+   - 2010: $5,700.35 billion
+   - 2011: $6,157.47 billion
+   - 2012: $6,203.21 billion
+   - 2013: $5,155.72 billion
+   - 2014: $4,616.52 billion
+
+4. South Korea:
+   - 2010: $1,464.26 billion
+   - 2011: $1,622.03 billion
+   - 2012: $1,624.76 billion
+   - 2013: $1,305.76 billion
+   - 2014: $1,411.25 billion
+
+5. Indonesia:
+   - 2010: $706.39 billion
+   - 2011: $846.48 billion
+   - 2012: $878.47 billion
+   - 2013: $868.36 billion
+   - 2014: $891.77 billion
+```
+
+#### Prompt After 1 Interation
+
+```
+ROLE
+
+You are an economic analyst.
+
+TASK
+
+Your task is to calculate the annual percentage change in GDP for each country based on the provided data.
+
+REQUIREMENTS_AND_RESTRICTIONS
+
+- The data will be provided in the format: "Year: $GDP in billions"
+- Calculate the percentage change from year to year for each country.
+- Present the results in a table format, with each row representing the change from one year to the next, and each column representing a different country.
+- The table should be formatted as "Year-Year,Country1,Country2,..."
+- The percentage change should be calculated as ((GDP Year 2 - GDP Year 1) / GDP Year 1) * 100
+- The percentage change should be rounded to two decimal places and followed by a "%" symbol.
+- If data for a year is missing for a country, leave that cell blank in the table.
+```
+
+## License
+
+This project is licensed under the MIT License. Please see the [LICENSE](LICENSE) file for more information.
+
+## Contact
+
+For any questions or feedback regarding this project, please feel free to reach out to Yale Huang at [email protected].
+
 ---
-title: Meta Prompt Docker
-emoji: 🦀
-colorFrom: red
-colorTo: purple
-sdk: docker
-pinned: false
----
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+**Acknowledgements:**
+
+I would like to express my gratitude to my colleagues at [Wiz.AI](https://www.wiz.ai/) for their support and contributions.

app/config.py

@@ -0,0 +1,18 @@
+# config.py
+from confz import BaseConfig
+from pydantic import BaseModel, Extra
+from typing import Optional
+
+class LLMConfig(BaseModel):
+    type: str
+
+    class Config:
+        extra = Extra.allow
+
+class MetaPromptConfig(BaseConfig):
+    llms: Optional[dict[str, LLMConfig]]
+    examples_path: Optional[str]
+    server_name: Optional[str] = None
+    server_port: Optional[int] = None
+    recursion_limit: Optional[int] = 25
+    recursion_limit_max: Optional[int] = 50

app/examples/log.csv

@@ -0,0 +1,236 @@
+User Message,Expected Output,Acceptance Criteria
+How do I reverse a list in Python?,Use the `[::-1]` slicing technique or the `list.reverse()` method.,"Similar in meaning, text length and style."
+(2+8)*3,"(2+8)*3
+= 10*3
+= 30
+","
+* Exactly text match.
+* Acceptable differences:
+  * Extra or missing spaces.
+  * Extra or missing line breaks at the beginning or end of the output.
+"
+"Here is the GDP data in billions of US dollars (USD) for these years:
+
+Germany:
+
+2015: $3,368.29 billion
+2016: $3,467.79 billion
+2017: $3,677.83 billion
+2018: $3,946.00 billion
+2019: $3,845.03 billion
+France:
+
+2015: $2,423.47 billion
+2016: $2,465.12 billion
+2017: $2,582.49 billion
+2018: $2,787.86 billion
+2019: $2,715.52 billion
+United Kingdom:
+
+2015: $2,860.58 billion
+2016: $2,650.90 billion
+2017: $2,622.43 billion
+2018: $2,828.87 billion
+2019: $2,829.21 billion
+Italy:
+
+2015: $1,815.72 billion
+2016: $1,852.50 billion
+2017: $1,937.80 billion
+2018: $2,073.90 billion
+2019: $1,988.14 billion
+Spain:
+
+2015: $1,199.74 billion
+2016: $1,235.95 billion
+2017: $1,313.13 billion
+2018: $1,426.19 billion
+2019: $1,430.38 billion
+","Year,Germany,France,United Kingdom,Italy,Spain
+2016-2015,2.96%,1.71%,-7.35%,2.02%,3.04%
+2017-2016,5.08%,4.78%,-1.07%,4.61%,6.23%
+2018-2017,7.48%,7.99%,7.89%,7.10%,8.58%
+2019-2018,-2.56%,-2.59%,0.01%,-4.11%,0.30%
+","
+* Strict text matching of the header row and first column(year).
+* Acceptable differences:
+  * Differences in digital/percentage values in the table, even significant ones.
+  * Extra or missing spaces.
+  * Extra or missing line breaks.
+"
+"Gene sequence: ATGGCCATGGCGCCCAGAACTGAGATCAATAGTACCCGTATTAACGGGTGA
+Species: Escherichia coli","{
+  ""Gene Sequence Analysis Results"": {
+    ""Basic Information"": {
+      ""Sequence Length"": 54,
+      ""GC Content"": ""51.85%""
+    },
+    ""Nucleotide Composition"": {
+      ""A"": {""Count"": 12, ""Percentage"": ""22.22%""},
+      ""T"": {""Count"": 11, ""Percentage"": ""20.37%""},
+      ""G"": {""Count"": 16, ""Percentage"": ""29.63%""},
+      ""C"": {""Count"": 15, ""Percentage"": ""27.78%""}
+    },
+    ""Codon Analysis"": {
+      ""Start Codon"": ""ATG"",
+      ""Stop Codon"": ""TGA"",
+      ""Codon Table"": [
+        {""Codon"": ""ATG"", ""Amino Acid"": ""Methionine"", ""Position"": 1},
+        {""Codon"": ""GCC"", ""Amino Acid"": ""Alanine"", ""Position"": 2},
+        {""Codon"": ""ATG"", ""Amino Acid"": ""Methionine"", ""Position"": 3},
+        // ... other codons ...
+        {""Codon"": ""TGA"", ""Amino Acid"": ""Stop Codon"", ""Position"": 18}
+      ]
+    },
+    ""Potential Function Prediction"": {
+      ""Protein Length"": 17,
+      ""Possible Functional Domains"": [
+        {""Domain Name"": ""ABC Transporter"", ""Start Position"": 5, ""End Position"": 15, ""Confidence"": ""75%""},
+        {""Domain Name"": ""Membrane Protein"", ""Start Position"": 1, ""End Position"": 17, ""Confidence"": ""60%""}
+      ],
+      ""Secondary Structure Prediction"": {
+        ""α-helix"": [""2-8"", ""12-16""],
+        ""β-sheet"": [""9-11""],
+        ""Random Coil"": [""1"", ""17""]
+      }
+    },
+    ""Homology Analysis"": {
+      ""Most Similar Sequences"": [
+        {
+          ""Gene Name"": ""abcT"",
+          ""Species"": ""Salmonella enterica"",
+          ""Similarity"": ""89%"",
+          ""E-value"": ""3e-25""
+        },
+        {
+          ""Gene Name"": ""yojI"",
+          ""Species"": ""Escherichia coli"",
+          ""Similarity"": ""95%"",
+          ""E-value"": ""1e-30""
+        }
+      ]
+    },
+    ""Mutation Analysis"": {
+      ""SNP Sites"": [
+        {""Position"": 27, ""Wild Type"": ""A"", ""Mutant"": ""G"", ""Amino Acid Change"": ""Glutamine->Arginine""},
+        {""Position"": 42, ""Wild Type"": ""C"", ""Mutant"": ""T"", ""Amino Acid Change"": ""None (Synonymous Mutation)""}
+      ]
+    }
+  }
+}","* Consistent with Expected Output:
+  * Formats of all JSON sections
+  * Data types of all JSON fields
+  * Top layer sections
+* Acceptable differences:
+  * Extra or missing spaces
+  * Extra or missing line breaks at the beginning or end of the output
+  * Differences in JSON field values
+  * JSON wrapped in backquotes"
+今天下午3点，在北京国家会议中心，阿里巴巴集团董事局主席马云宣布将投资100亿元人民币用于农村电商发展。这一决定受到了与会代表的热烈欢迎，大家认为这将为中国农村经济带来新的机遇。,"{
+  ""文本分析结果"": {
+    ""情感分析"": {
+      ""整体情感"": ""积极"",
+      ""情感得分"": 0.82,
+      ""情感细分"": {
+        ""乐观"": 0.75,
+        ""兴奋"": 0.60,
+        ""期待"": 0.85
+      }
+    },
+    ""实体识别"": [
+      {""实体"": ""北京"", ""类型"": ""地点"", ""起始位置"": 7, ""结束位置"": 9},
+      {""实体"": ""国家会议中心"", ""类型"": ""地点"", ""起始位置"": 9, ""结束位置"": 15},
+      {""实体"": ""阿里巴巴集团"", ""类型"": ""组织"", ""起始位置"": 16, ""结束位置"": 22},
+      {""实体"": ""马云"", ""类型"": ""人物"", ""起始位置"": 26, ""结束位置"": 28},
+      {""实体"": ""100亿元"", ""类型"": ""金额"", ""起始位置"": 32, ""结束位置"": 37},
+      {""实体"": ""人民币"", ""类型"": ""货币"", ""起始位置"": 37, ""结束位置"": 40},
+      {""实体"": ""中国"", ""类型"": ""地点"", ""起始位置"": 71, ""结束位置"": 73}
+    ],
+    ""关键词提取"": [
+      {""关键词"": ""农村电商"", ""权重"": 0.95},
+      {""关键词"": ""马云"", ""权重"": 0.85},
+      {""关键词"": ""投资"", ""权重"": 0.80},
+      {""关键词"": ""阿里巴巴"", ""权重"": 0.75},
+      {""关键词"": ""经济机遇"", ""权重"": 0.70}
+    ]
+  }
+}","* Consistent with Expected Output:
+  * Formats of all JSON sections
+  * Data types of all JSON fields
+  * Top layer sections
+* Acceptable differences:
+  * Differences in digital values in the table.
+  * Extra or missing spaces.
+  * Extra or missing line breaks at the beginning or end of the output.
+  * Differences in JSON field values
+  * Differences in section/item orders.
+  * JSON wrapped in backquotes."
+Low-noise amplifier,"A '''low-noise amplifier''' ('''LNA''') is an electronic component that amplifies a very low-power [[signal]] without significantly degrading its [[signal-to-noise ratio]] (SNR). Any [[electronic amplifier]] will increase the power of both the signal and the [[Noise (electronics)|noise]] present at its input, but the amplifier will also introduce some additional noise. LNAs are designed to minimize that additional noise, by choosing special components, operating points, and [[Circuit topology (electrical)|circuit topologies]]. Minimizing additional noise must balance with other design goals such as [[power gain]] and [[impedance matching]].
+
+LNAs are found in [[Radio|radio communications]] systems, [[Amateur Radio]] stations, medical instruments and [[electronic test equipment]]. A typical LNA may supply a power gain of 100 (20 [[decibels]] (dB)) while decreasing the SNR by less than a factor of two (a 3 dB [[noise figure]] (NF)). Although LNAs are primarily concerned with weak signals that are just above the [[noise floor]], they must also consider the presence of larger signals that cause [[intermodulation distortion]].","* Consistent with Expected Output:
+  * Language
+  * Text length
+  * Text style
+  * Text structures
+* Cover all the major content of Expected Output.
+* Acceptable differences:
+  * Minor format differences.
+  * Expression differences.
+  * Numerical differences.
+  * Additional content in Actual Output.
+  * Missing minor content in Actual Output."
+What is the meaning of life?,"[
+  {""persona"": ""Philosopher"", ""prompt"": ""Explore the concept of life's meaning through the lens of existentialism and purpose-driven existence.""},
+  {""persona"": ""Scientist"", ""prompt"": ""Examine the biological and evolutionary perspectives on the function and significance of life.""},
+  {""persona"": ""Child"", ""prompt"": ""Imagine you're explaining to a curious 7-year-old what makes life special and important.""}
+]","* Consistent with Expected Output:
+  * Formats of all JSON sections
+  * Data types and formats of all JSON fields
+  * Top layer sections
+* Acceptable differences:
+  * Different personas or prompts
+  * Different numbers of personas
+  * Extra or missing spaces
+  * Extra or missing line breaks at the beginning or end of the output
+* Unacceptable:
+  * Showing the personas in Expected Output in System Message"
+"<?php
+$username = $_POST['username'];
+$password = $_POST['password'];
+
+$query = ""SELECT * FROM users WHERE username = '$username' AND password = '$password'"";
+$result = mysqli_query($connection, $query);
+
+if (mysqli_num_rows($result) > 0) {
+    echo ""Login successful"";
+} else {
+    echo ""Login failed"";
+}
+?>","security_analysis:
+  vulnerabilities:
+    - type: SQL Injection
+      severity: Critical
+      description: Unsanitized user input directly used in SQL query
+      mitigation: Use prepared statements or parameterized queries
+    - type: Password Storage
+      severity: High
+      description: Passwords stored in plain text
+      mitigation: Use password hashing (e.g., bcrypt) before storage
+  additional_issues:
+    - Lack of input validation
+    - No CSRF protection
+    - Potential for timing attacks in login logic
+  overall_risk_score: 9.5/10
+  recommended_actions:
+    - Implement proper input sanitization
+    - Use secure password hashing algorithms
+    - Add CSRF tokens to forms
+    - Consider using a secure authentication library","* Consistent with Expected Output:
+  * Formats of all YAML sections
+  * Data types and formats of all YAML fields
+  * Top layer sections
+* Acceptable differences:
+  * Differences in field values
+  * Extra or missing spaces
+  * Extra or missing line breaks at the beginning or end of the output
+  * YAML wrapped in backquotes"

app/gradio_meta_prompt.py

@@ -0,0 +1,103 @@
+import gradio as gr
+from confz import BaseConfig, CLArgSource, EnvSource, FileSource
+from meta_prompt import MetaPromptGraph, AgentState
+from langchain_openai import ChatOpenAI
+from app.config import MetaPromptConfig
+
+class LLMModelFactory:
+    def __init__(self):
+        pass
+
+    def create(self, model_type: str, **kwargs):
+        model_class = globals()[model_type]
+        return model_class(**kwargs)
+    
+llm_model_factory = LLMModelFactory()
+
+def process_message(user_message, expected_output, acceptance_criteria, initial_system_message,
+                    recursion_limit: int, model_name: str):
+    # Create the input state
+    input_state = AgentState(
+        user_message=user_message,
+        expected_output=expected_output,
+        acceptance_criteria=acceptance_criteria,
+        system_message=initial_system_message
+    )
+    
+    # Get the output state from MetaPromptGraph
+    type = config.llms[model_name].type
+    args = config.llms[model_name].model_dump(exclude={'type'})
+    llm = llm_model_factory.create(type, **args)
+    meta_prompt_graph = MetaPromptGraph(llms=llm)
+    output_state = meta_prompt_graph(input_state, recursion_limit=recursion_limit)
+    
+    # Validate the output state
+    system_message = ''
+    output = ''
+    analysis = ''
+
+    if 'best_system_message' in output_state and output_state['best_system_message'] is not None:
+        system_message = output_state['best_system_message']
+    else:
+        system_message = "Error: The output state does not contain a valid 'best_system_message'"
+
+    if 'best_output' in output_state and output_state['best_output'] is not None:
+        output = output_state["best_output"]
+    else:
+        output = "Error: The output state does not contain a valid 'best_output'"
+
+    if 'analysis' in output_state and output_state['analysis'] is not None:
+        analysis = output_state['analysis']
+    else:
+        analysis = "Error: The output state does not contain a valid 'analysis'"
+
+    return system_message, output, analysis
+
+class FileConfig(BaseConfig):
+    config_file: str = 'config.yml'  # default path
+
+pre_config_sources = [
+    EnvSource(prefix='METAPROMPT_', allow_all=True),
+    CLArgSource()
+]
+pre_config = FileConfig(config_sources=pre_config_sources)
+
+config_sources = [
+    FileSource(file=pre_config.config_file, optional=True),
+    EnvSource(prefix='METAPROMPT_', allow_all=True),
+    CLArgSource()
+]
+
+config = MetaPromptConfig(config_sources=config_sources)
+
+# Create the Gradio interface
+iface = gr.Interface(
+    fn=process_message,
+    inputs=[
+        gr.Textbox(label="User Message", show_copy_button=True),
+        gr.Textbox(label="Expected Output", show_copy_button=True),
+        gr.Textbox(label="Acceptance Criteria", show_copy_button=True),
+    ],
+    outputs=[
+        gr.Textbox(label="System Message", show_copy_button=True),
+        gr.Textbox(label="Output", show_copy_button=True),
+        gr.Textbox(label="Analysis", show_copy_button=True)
+    ],
+    additional_inputs=[
+        gr.Textbox(label="Initial System Message", show_copy_button=True, value=""),
+        gr.Number(label="Recursion Limit", value=config.recursion_limit,
+                  precision=0, minimum=1, maximum=config.recursion_limit_max, step=1),
+        gr.Dropdown(
+            label="Model Name",
+            choices=config.llms.keys(),
+            value=list(config.llms.keys())[0],
+        )
+    ],
+    # stop_btn = gr.Button("Stop", variant="stop", visible=True),
+    title="MetaPromptGraph Chat Interface",
+    description="A chat interface for MetaPromptGraph to process user inputs and generate system messages.",
+    examples=config.examples_path
+)
+
+# Launch the Gradio app
+iface.launch(server_name=config.server_name, server_port=config.server_port)

config.yml

@@ -0,0 +1,39 @@
+llms:
+  groq/llama3-70b-8192:
+    type: ChatOpenAI
+    temperature: 0.1
+    model_name: "llama3-70b-8192"
+    # openai_api_key: ""
+    openai_api_base: "https://api.groq.com/openai/v1"
+    max_tokens: 8192
+    verbose: true
+  # anthropic/claude-3-haiku:
+  #   type: ChatOpenAI
+  #   temperature: 0.1
+  #   model_name: "anthropic/claude-3-haiku:beta"
+  #   openai_api_key: ""
+  #   openai_api_base: "https://openrouter.ai/api/v1"
+  #   max_tokens: 8192
+  #   verbose: true
+  # anthropic/claude-3-sonnet:
+  #   type: ChatOpenAI
+  #   temperature: 0.1
+  #   model_name: "anthropic/claude-3-sonnet:beta"
+  #   openai_api_key: ""
+  #   openai_api_base: "https://openrouter.ai/api/v1"
+  #   max_tokens: 8192
+  #   verbose: true
+  # anthropic/deepseek-chat:
+  #   type: ChatOpenAI
+  #   temperature: 0.1
+  #   model_name: "deepseek/deepseek-chat"
+  #   openai_api_key: ""
+  #   openai_api_base: "https://openrouter.ai/api/v1"
+  #   max_tokens: 8192
+  #   verbose: true
+
+examples_path: "app/examples"
+# server_name: 0.0.0.0
+# server_port: 7860
+recursion_limit: 16
+recursion_limit_max: 20

demo/cot_meta_prompt.ipynb

@@ -0,0 +1,69 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "v98KGZUT17EJ"
+      },
+      "outputs": [],
+      "source": [
+        "!wget https://github.com/yaleh/meta-prompt/raw/main/prompt_ui.py\n",
+        "!wget https://github.com/yaleh/meta-prompt/raw/main/default_meta_prompts.py"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MO89Z8-UY5Ht"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install gradio langchain scikit-learn openai"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9BZL20lBYLbj"
+      },
+      "outputs": [],
+      "source": [
+        "import openai\n",
+        "import os\n",
+        "openai.api_key = ''\n",
+        "os.environ[\"OPENAI_API_KEY\"] = openai.api_key\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z8M3eFzXZaOb"
+      },
+      "outputs": [],
+      "source": [
+        "from prompt_ui import PromptUI\n",
+        "\n",
+        "app = PromptUI()\n",
+        "app.ui.queue().launch(debug=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

demo/cot_meta_prompt.py

@@ -0,0 +1,61 @@
+"""
+MIT License
+
+Copyright (c) 2023 Yale Huang
+Email: [email protected]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+"""
+
+import argparse
+import os
+import openai
+
+import gradio as gr
+from demo.prompt_ui import PromptUI
+
+class ChatbotApp:
+    def __init__(self, args):
+        if args.api_key:
+            os.environ["OPENAI_API_KEY"] = args.api_key
+
+        if args.openai_api_base:
+            os.environ["OPENAI_API_BASE"] = args.openai_api_base
+
+        if args.proxy:
+            os.environ["OPENAI_PROXY"] = args.proxy
+
+        self.prompt_ui = PromptUI(advanced_mode=args.advanced_mode)
+
+        self.ui = gr.TabbedInterface(
+            [self.prompt_ui.ui], 
+            ['Prompt']
+        )
+    def launch(self, *args, **kwargs):
+        self.ui.launch(*args, **kwargs)
+    
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--api_key", type=str, required=True, help="OpenAI API key")
+    parser.add_argument("--proxy", type=str, default=None, help="Proxy settings")
+    parser.add_argument("--share", action='store_true',
+                        help="Launch app with sharing option")
+    parser.add_argument("--advanced_mode", action='store_true', default=False,
+                        help="Enable advanced mode")
+    parser.add_argument("--server_name", type=str, default="127.0.0.1", help="Server name or IP address")
+    parser.add_argument("--openai_api_base", type=str, default=None, help="OpenAI API base URL")
+ 
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    args = parse_args()
+    app = ChatbotApp(args)
+    app.launch(share=args.share, server_name=args.server_name)

demo/default_meta_prompts.py

@@ -0,0 +1,219 @@
+DEFAULT_META_SYSTEM_PROMPT = \
+'''
+You are a Prompt Engineer. You review the Prompt template for GTP-3.5 and suggest changes.
+
+# Prompt template format
+
+You require Prompt to be written in the following format:
+
+```
+<ROLE>
+
+<TASK>
+
+<REQUIREMENTS_AND_RESTRICTIONS>
+
+```
+
+* ROLE: The role the LLM is required to play. Describe it in one sentence.
+* TASK: A summary and overall description of the tasks to be performed by LLM. Describe it in one or more sentences.
+* REQUIREMENTS_AND_RESTRICTIONS: Specific requirements for the task. Describe using Markdown List.
+
+A string of user message [USER_MESSAGE] entered by the user will be attached to the end of the prompt.
+
+# Check input
+
+Check the input format as follows:
+
+```
+* Prompt Template
+
+[PROMPT_TEMPLATE]
+
+* User Message
+
+[USER_MESSAGE]
+
+* Expected GPT Message
+
+[EXPECTED_GPT_MESSAGE]
+
+* GPT Message
+
+[GPT_MESSAGE]
+```
+
+* PROMPT_TEMPLATE: Prompt template that conforms to the above Prompt template format.
+* USER_MESSAGE: User input. Used to replace {user_message} in the Prompt template.
+* EXPECTED_GPT_MESSAGE: Expect output generated by GPT.
+* GPT_MESSAGE: GPT is actually based on the output generated by PROMPT_TEMPLATE and USER_MESSAGE.
+
+# examine
+
+Check and recommend modifying the Prompt template as follows to produce output closer to EXPECTED_GPT_MESSAGE:
+
+* Read and parse PROMPT_TEMPLATE, USER_MESSAGE and EXPECTED_GPT_MESSAGE.
+   * Generate a description [TD] of this task according to your understanding.
+   * Analyze the correlation between PROMPT_TEMPLATE and USER_MESSAGE [UMR].
+   * Analyze and describe the characteristics of EXPECTED_GPT_MESSAGE in terms of text length, format, content, meaning and style.
+   * Analyze whether PROMPT_TEMPLATE and EXPECTED_GPT_MESSAGE match and list the differences [PED].
+* Check whether GPT_MESSAGE conforms to EXPECTED_GPT_MESSAGE. Refer to EXPECTED_GPT_MESSAGE and TD analysis on how GPT_MESSAGE can be optimized to be close to EXPECTED_GPT_MESSAGE. Modification suggestions are listed in detail [MCSL].
+   * Pay attention to checking the text length, format, content, meaning and style, and output corresponding modification suggestions.
+     * Suggested modifications to text length should include quantitative numerical descriptions.
+     * Suggestions for changes to text formatting should include specific examples enclosed by "```".
+   * Pay attention to check whether unnecessary content is included in GPT_MESSAGE and output corresponding modification suggestions.
+   * Suggestions for modifying local content should include the modifiable fragments and recommended modified fragments in GPT_MESSAGE.
+* Check PROMPT_TEMPLATE: Analyze and list suggestions [CSL] for how to modify PROMPT_TEMPLATE to produce output closer to EXPECTED_GPT_MESSAGE.
+   * For requirements that have been stated in REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE, they should also be emphasized in TASK, and the opposite tendency (such as reverse adjustment of quantitative indicators or style descriptions) should be emphasized punitively to construct the strongest Negative feedback***.
+    * For format requirements that have been stated in REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE, add an example enclosed with "```".
+   * Based on PED recommendations on how to modify PROMPT_TEMPLATE.
+   * Analyze and suggest how to modify PROMPT_TEMPLATE to implement the MCSL listed above.
+   * Analyze whether PROMPT_TEMPLATE conforms to the format defined by `Prompt template format` and suggest how to modify it.
+   * Analyze those instructions that do not comply with EXPECTED_GPT_MESSAGE and are clearly misleading, and recommend modifications.
+   * Modifications to PROMPT_TEMPLATE should not introduce more information related to USER_MESSAGE.
+   * In TASK and REQUIREMENTS_AND_RESTRICTIONS, group the requirements for the same content together.
+   * If there are multiple steps, use a numbered list to list the steps clearly.
+   * Care should be taken to avoid unnecessary changes, and the original text should be retained as much as possible for parts that do not need to be changed.
+   * Only output [CSL], do not output the modified PROMPT_TEMPLATE.
+* Check and filter the Change Suggestions List [CSL] for information related to USER_MESSAGE.
+   * Only output the filtered modification suggestion list [RCSL], do not output the modified PROMPT_TEMPLATE.
+* Execute the above filtered modification suggestion list [RCSL] and ***output the modified PROMPT_TEMPLATE***.
+  * Execute RCSL only, avoid other changes.
+  * Care should be taken to avoid unnecessary changes, and the original text should be retained as much as possible for parts that do not need to be changed, except the requirements that have been stated in TASK or REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE.
+  * Strictly use the following format for output:
+```
+<!-- BEGIN OF PROMPT -->
+
+<Updated Prompt>
+
+<!-- END OF PROMPT -->
+```
+  * If there's no change, output following fixed message instead:
+```
+<!-- NO CHANGE TO PROMPT -->
+```
+* Evaluation modified PROMPT_TEMPLATE.
+   * Analyze the changes it may cause in the output of LLM [EC].
+   * Analyze whether EC would be more consistent with EXPECTED_GPT_MESSAGE.
+   * Analyze the correlation between modified PROMPT_TEMPLATE and USER_MESSAGE [UMRC].
+   * Analyze UMR and UMRC to determine whether the modification introduces additional information about USER_MESSAGE. If introduced, issue a warning.
+* NOTICE: During the above steps, ****output RCSL and the modified PROMPT_TEMPLATE only, don't print the output of other steps***.
+
+----
+
+Now, provide the PROMPT_TEMPLATE, USER_MESSAGE, EXPECTED_GPT_MESSAGE, and GPT_MESSAGE for review.
+
+'''
+
+DEFAULT_META_SYSTEM_PROMPT_WITH_OTHER_PROMPTS = \
+'''
+You are a Prompt Engineer. You review the Prompt template for GTP-3.5 and suggest changes.
+
+# Prompt template format
+
+You require Prompt to be written in the following format:
+
+```
+<ROLE>
+
+<TASK>
+
+<REQUIREMENTS_AND_RESTRICTIONS>
+
+```
+
+* ROLE: The role the LLM is required to play. Describe it in one sentence.
+* TASK: A summary and overall description of the tasks to be performed by LLM. Describe it in one or more sentences.
+* REQUIREMENTS_AND_RESTRICTIONS: Specific requirements for the task. Describe using Markdown List.
+
+A string of user message [USER_MESSAGE] entered by the user will be attached to the end of the prompt.
+
+# Check input
+
+Check the input format as follows:
+
+```
+* Prompt Template
+
+[PROMPT_TEMPLATE]
+
+* User Message
+
+[USER_MESSAGE]
+
+* Other User Messages
+
+[OTHER_USER_MESSAGES]
+
+* Expected GPT Message
+
+[EXPECTED_GPT_MESSAGE]
+
+* GPT Message
+
+[GPT_MESSAGE]
+```
+
+* PROMPT_TEMPLATE: Prompt template that conforms to the above Prompt template format.
+* USER_MESSAGE: User input. Used to replace {user_message} in the Prompt template.
+* OTHER_USER_MESSAGES: Other user messages that the prompt template is expected to be compatible with.
+* EXPECTED_GPT_MESSAGE: Expect output generated by GPT.
+* GPT_MESSAGE: GPT is actually based on the output generated by PROMPT_TEMPLATE and USER_MESSAGE.
+
+# examine
+
+Check and recommend modifying the Prompt template as follows to produce output closer to EXPECTED_GPT_MESSAGE:
+
+* Read and parse PROMPT_TEMPLATE, USER_MESSAGE, OTHER_USER_MESSAGES and EXPECTED_GPT_MESSAGE.
+   * Generate a description [TD] of this task according to your understanding.
+   * Analyze the correlation between PROMPT_TEMPLATE, USER_MESSAGE and OTHER_USER_MESSAGES [UMR].
+   * Analyze and describe the characteristics of EXPECTED_GPT_MESSAGE in terms of text length, format, content, meaning and style.
+   * Analyze whether PROMPT_TEMPLATE and EXPECTED_GPT_MESSAGE match and list the differences [PED].
+* Check whether GPT_MESSAGE conforms to EXPECTED_GPT_MESSAGE. Refer to EXPECTED_GPT_MESSAGE and TD analysis on how GPT_MESSAGE can be optimized to be close to EXPECTED_GPT_MESSAGE. Modification suggestions are listed in detail [MCSL].
+   * Pay attention to checking the text length, format, content, meaning and style, and output corresponding modification suggestions.
+     * Suggested modifications to text length should include quantitative numerical descriptions.
+     * Suggestions for changes to text formatting should include specific examples enclosed by "```".
+   * Pay attention to check whether unnecessary content is included in GPT_MESSAGE and output corresponding modification suggestions.
+   * Suggestions for modifying local content should include the modifiable fragments and recommended modified fragments in GPT_MESSAGE.
+* Check PROMPT_TEMPLATE: Analyze and list suggestions [CSL] for how to modify PROMPT_TEMPLATE to produce output closer to EXPECTED_GPT_MESSAGE.
+   * For requirements that have been stated in REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE, they should also be emphasized in TASK, and the opposite tendency (such as reverse adjustment of quantitative indicators or style descriptions) should be emphasized punitively to construct the strongest Negative feedback***.
+    * For format requirements that have been stated in REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE, add an example enclosed with "```".
+   * Based on PED recommendations on how to modify PROMPT_TEMPLATE.
+   * Analyze and suggest how to modify PROMPT_TEMPLATE to implement the MCSL listed above.
+   * Analyze whether PROMPT_TEMPLATE conforms to the format defined by `Prompt template format` and suggest how to modify it.
+   * Analyze those instructions that do not comply with EXPECTED_GPT_MESSAGE and are clearly misleading, and recommend modifications.
+   * Modifications to PROMPT_TEMPLATE should not introduce more information related to USER_MESSAGE.
+   * In TASK and REQUIREMENTS_AND_RESTRICTIONS, group the requirements for the same content together.
+   * If there are multiple steps, use a numbered list to list the steps clearly.
+   * Care should be taken to avoid unnecessary changes, and the original text should be retained as much as possible for parts that do not need to be changed.
+   * Only output [CSL], do not output the modified PROMPT_TEMPLATE.
+* Check and filter the Change Suggestions List [CSL] for information related to USER_MESSAGE.
+   * Only output the filtered modification suggestion list [RCSL], do not output the modified PROMPT_TEMPLATE.
+   * Keep it compatible with OTHER_USER_MESSAGES.
+* Execute the above filtered modification suggestion list [RCSL] and ***output the modified PROMPT_TEMPLATE***.
+  * Execute RCSL only, avoid other changes.
+  * Care should be taken to avoid unnecessary changes, and the original text should be retained as much as possible for parts that do not need to be changed, except the requirements that have been stated in TASK or REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE.
+  * Strictly use the following format for output:
+```
+<!-- BEGIN OF PROMPT -->
+
+<Updated Prompt>
+
+<!-- END OF PROMPT -->
+```
+  * If there's no change, output following fixed message instead:
+```
+<!-- NO CHANGE TO PROMPT -->
+```
+* Evaluation modified PROMPT_TEMPLATE.
+   * Analyze the changes it may cause in the output of LLM [EC].
+   * Analyze whether EC would be more consistent with EXPECTED_GPT_MESSAGE.
+   * Analyze the correlation between modified PROMPT_TEMPLATE, USER_MESSAGE and OTHER_USER_MESSAGES [UMRC].
+   * Analyze UMR and UMRC to determine whether the modification introduces additional information about USER_MESSAGE. If introduced, issue a warning.
+* NOTICE: During the above steps, ****output RCSL and the modified PROMPT_TEMPLATE only, don't print the output of other steps***.
+
+----
+
+Now, provide the PROMPT_TEMPLATE, USER_MESSAGE, OTHER_USER_MESSAGES, EXPECTED_GPT_MESSAGE, and GPT_MESSAGE for review.
+
+'''

demo/prompt_ui.py

@@ -0,0 +1,712 @@
+"""
+MIT License
+
+Copyright (c) 2023 Yale Huang
+Email: [email protected]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+"""
+
+import re
+import gradio as gr
+
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from demo.default_meta_prompts import *
+
+gpt_models_not_legacy = [
+    "gpt-4",
+    "gpt-4-0613",
+    "gpt-4-32k",
+    "gpt-4-32k-0613",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-16k",
+    "gpt-3.5-turbo-instruct",
+    "gpt-3.5-turbo-0613"
+]
+
+gpt_models_legacy = [
+    "gpt-3.5-turbo-instruct",
+    "gpt-3.5-turbo-0301",
+    "text-davinci-003",
+    "text-davinci-002",
+    "code-davinci-002"
+]
+
+DEFAULT_MODEL_FOR_GENERATING="gpt-4"
+DEFAULT_MODEL_FOR_TESTING="gpt-3.5-turbo"
+DEFAULT_MODEL_FOR_OUTPUT_EVALUATING="gpt-3.5-turbo-instruct"
+DEFAULT_CURRENT_SYSTEM_PROMPT = ''
+DEFAULT_OUTPUT_EVALUATING_PROMPT = 'Find out which is more similar to string S, A or B? Print nothing if there\'s no significant difference between A and B. Else, print the result (letter A or B) only. Do nothing else.'
+
+class PromptUI:
+    def __init__(self, advanced_mode = False, enable_other_user_prompts = False):
+        self.advanced_mode = advanced_mode
+        self.enable_other_user_prompts = enable_other_user_prompts
+        self.ui = self.init_ui()
+
+    def init_ui(self):
+        with gr.Blocks() as prompt_ui:            
+            with gr.Row():
+                with gr.Column():
+                    self.testing_user_prompt_textbox = gr.Textbox(
+                        label="Testing User Prompt", 
+                        lines=10, 
+                        interactive=True,
+                        show_copy_button=True
+                        )
+                    self.expect_output_textbox = gr.Textbox(
+                        label="Expected Output", 
+                        lines=5, 
+                        interactive=True,
+                        show_copy_button=True
+                        )
+                    self.other_user_prompts_checkbox = gr.Checkbox(
+                        label="Other User Prompts", 
+                        info="Enable other user prompts in meta prompt?",
+                        value=self.enable_other_user_prompts
+                        )
+                    self.other_user_prompts_textbox = gr.Textbox(
+                        label="Other User Prompts",
+                        lines=10, 
+                        interactive=True,
+                        placeholder="Wrap each prompt with a pair of '```'.",
+                        visible=self.enable_other_user_prompts,
+                        show_copy_button=True
+                        )
+                    # Add gr.Number here for iterations input
+                    self.iterations_number = gr.Number(value=1, label="Optimize Iterations", min=1, max=1000, step=1, decimals=0)
+                    # Add button to trigger optimization here
+                    self.optimize_btn = gr.Button(value="Optimize Prompt", variant='primary')
+                    self.similar_candidate_textbox = gr.Textbox(label="Similarity Delta", lines=1, interactive=True)
+                    self.compare_outputs_btn = gr.Button(value="Compare Outputs")
+
+                with gr.Column():
+                    self.new_system_prompt_textbox = gr.Textbox(
+                        label="New System Prompt", 
+                        lines=5, 
+                        interactive=True,
+                        show_copy_button=True
+                        )
+                    self.new_output_textbox = gr.Textbox(
+                        label="New Output", 
+                        lines=5, 
+                        interactive=True,
+                        show_copy_button=True
+                        )
+                    with gr.Row():
+                        self.run_meta_btn = gr.Button(value="↑ Single Step Optimize")
+                        self.run_new_btn = gr.Button(value="⟳ Run New")
+                    self.new_system_prompt_changed = gr.Checkbox(
+                        label="New System Prompt Changed",
+                        value=False,
+                        interactive=False
+                        )
+
+                with gr.Column():
+                    self.current_system_prompt_textbox = gr.Textbox(
+                        label="Current System Prompt",
+                        value=DEFAULT_CURRENT_SYSTEM_PROMPT,
+                        lines=5,
+                        interactive=True,
+                        show_copy_button=True
+                        )
+                    self.current_output_textbox = gr.Textbox(
+                        label="Current Output", 
+                        lines=5, 
+                        interactive=True,
+                        show_copy_button=True
+                        )
+                    with gr.Row():
+                        self.accept_new_btn = gr.Button(value="→ Accept New Prompt")
+                        self.run_current_btn = gr.Button(value="⟳ Run Current")
+
+            with gr.Row(visible=self.advanced_mode):
+                with gr.Column():
+                    self.meta_system_prompt_textbox = gr.Textbox(label="Meta System Prompt", 
+                                                                value=DEFAULT_META_SYSTEM_PROMPT,
+                                                                lines=10, 
+                                                                interactive=True
+                                                                )
+                with gr.Column():
+                    self.merged_meta_prompt_textbox = gr.Textbox(label="Merged Meta System Prompt", 
+                                                                lines=10, 
+                                                                interactive=False,
+                                                                show_copy_button=True
+                                                                )
+                    self.merge_prompt_btn = gr.Button(value="Merge Meta System Prompt")
+                    # self.chatgpt_output_textbox = gr.Textbox(label="Paste ChatGPT Output", 
+                    #                                         lines=10,
+                    #                                         interactive=True
+                    #                                         )
+                    # self.parse_chatgpt_output_btn = gr.Button(value="Parse ChatGPT Output")                                      
+
+            with gr.Row(visible=self.advanced_mode):
+                with gr.Column():
+                    self.llm_model_meta_dropdown = gr.Dropdown(
+                        label="Generating LLM Model", 
+                        choices=gpt_models_not_legacy,
+                        value=DEFAULT_MODEL_FOR_GENERATING,
+                        interactive=True,
+                        allow_custom_value=False
+                    )
+
+                    self.llm_model_meta_temperature_slider = gr.Slider(
+                        minimum=0.0, 
+                        maximum=1.0, 
+                        step=0.01, 
+                        value=0.0,
+                        interactive=True,
+                        label="Generating LLM Model Temperature"
+                    )
+
+                    self.llm_model_meta_max_tokens_slider = gr.Slider(
+                        minimum=256, 
+                        maximum=32000, 
+                        step=256, 
+                        value=0,
+                        interactive=True, 
+                        label="Generating LLM Model Token Limit (0 for auto)"
+                    )
+
+                    self.llm_model_meta_request_timeout_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=600, 
+                        step=5, 
+                        value=600,
+                        interactive=True,
+                        label="Generating LLM Model Timeout"
+                    )
+
+                    self.llm_model_meta_max_retries_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=30, 
+                        step=1, 
+                        value=6,
+                        interactive=True,
+                        label="Generating LLM Model Max Retries"
+                    )
+
+                with gr.Column():
+                    self.llm_model_test_dropdown = gr.Dropdown(
+                        label="Testing LLM Model",
+                        choices=gpt_models_not_legacy,
+                        value=DEFAULT_MODEL_FOR_TESTING,
+                        interactive=True,
+                        allow_custom_value=False
+                    )
+
+                    self.llm_model_test_temperature_slider = gr.Slider(
+                        minimum=0.0,
+                        maximum=1.0, 
+                        step=0.01, 
+                        value=0.0,
+                        interactive=True,
+                        label="Testing LLM Model Temperature"
+                    )
+
+                    self.llm_model_test_max_tokens_slider = gr.Slider(
+                        minimum=256, 
+                        maximum=32000, 
+                        step=256, 
+                        value=0,
+                        interactive=True, 
+                        label="Testing LLM Model Token Limit  (0 for auto)"
+                    )
+
+                    self.llm_model_test_request_timeout_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=600, 
+                        step=5, 
+                        value=600, 
+                        interactive=True,
+                        label="Testing LLM Model Timeout"
+                    )
+
+                    self.llm_model_test_max_retries_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=30, 
+                        step=1, 
+                        value=6, 
+                        interactive=True,
+                        label="Testing LLM Model Max Retries"
+                    )
+                # with gr.Column():
+                #     self.llm_model_output_eval_dropdown = gr.Dropdown(label="Output Evaluating LLM Model",
+                #                                                       choices=gpt_models_legacy,
+                #                                                       value=DEFAULT_MODEL_FOR_OUTPUT_EVALUATING,
+                #                                                       interactive=True,
+                #                                                       allow_custom_value=False)
+                #     self.llm_model_output_eval_slider = gr.Slider(minimum=0.0,
+                #                                                          maximum=1.0,
+                #                                                          step=0.01,
+                #                                                          default=0.0, 
+                #                                                          label="Output Evaluating LLM Model of Temperature")
+
+
+            self.run_new_btn.click(
+                self.test_prompt, 
+                [
+                    self.new_system_prompt_textbox,
+                    self.testing_user_prompt_textbox,
+                    self.llm_model_test_dropdown,
+                    self.llm_model_test_max_retries_slider,
+                    self.llm_model_test_max_tokens_slider,
+                    self.llm_model_test_request_timeout_slider,
+                    self.llm_model_test_temperature_slider
+                ], 
+                [self.new_output_textbox]
+            )
+            self.run_current_btn.click(
+                self.test_prompt, 
+                [
+                    self.current_system_prompt_textbox, 
+                    self.testing_user_prompt_textbox,
+                    self.llm_model_test_dropdown,
+                    self.llm_model_test_max_retries_slider,
+                    self.llm_model_test_max_tokens_slider,
+                    self.llm_model_test_request_timeout_slider,
+                    self.llm_model_test_temperature_slider
+                ], 
+                [self.current_output_textbox]
+            )
+            self.run_meta_btn.click(
+                self.meta_prompt, 
+                [
+                    self.meta_system_prompt_textbox, 
+                    self.current_system_prompt_textbox,
+                    self.testing_user_prompt_textbox,
+                    self.other_user_prompts_textbox,
+                    self.expect_output_textbox,
+                    self.current_output_textbox,
+                    self.other_user_prompts_checkbox,
+                    self.llm_model_meta_dropdown,
+                    self.llm_model_meta_max_retries_slider,
+                    self.llm_model_meta_max_tokens_slider,
+                    self.llm_model_meta_request_timeout_slider,
+                    self.llm_model_meta_temperature_slider
+                ], 
+                [self.new_system_prompt_textbox, self.new_system_prompt_changed]
+            )
+            self.accept_new_btn.click(self.copy_new_prompts,
+                                      [self.new_system_prompt_textbox, self.new_output_textbox],
+                                      [self.current_system_prompt_textbox, self.current_output_textbox])
+            self.compare_outputs_btn.click(self.compare_outputs,
+                                           [self.new_output_textbox, self.current_output_textbox, self.expect_output_textbox],
+                                           [self.similar_candidate_textbox])
+            # Attach the optimize_prompt function to the button click event.
+            # You should implement this function according to your optimization logic.
+            self.optimize_btn.click(
+                self.optimize_prompt,
+                [
+                    self.meta_system_prompt_textbox, 
+                    self.current_system_prompt_textbox,
+                    self.testing_user_prompt_textbox,
+                    self.other_user_prompts_textbox,
+                    self.expect_output_textbox,
+                    self.current_output_textbox,
+                    self.iterations_number,
+                    self.other_user_prompts_checkbox,
+                    self.llm_model_meta_dropdown,
+                    self.llm_model_meta_max_retries_slider,
+                    self.llm_model_meta_max_tokens_slider,
+                    self.llm_model_meta_request_timeout_slider,
+                    self.llm_model_meta_temperature_slider,
+                    self.llm_model_test_dropdown,
+                    self.llm_model_test_max_retries_slider,
+                    self.llm_model_test_max_tokens_slider,
+                    self.llm_model_test_request_timeout_slider,
+                    self.llm_model_test_temperature_slider
+                ],
+                [self.new_system_prompt_textbox, self.new_system_prompt_changed])
+            
+            self.merge_prompt_btn.click(self.merge_meta_system_prompt,
+                                        [
+                                            self.meta_system_prompt_textbox, 
+                                            self.current_system_prompt_textbox,
+                                            self.other_user_prompts_textbox,
+                                            self.testing_user_prompt_textbox, 
+                                            self.expect_output_textbox, 
+                                            self.current_output_textbox,
+                                            self.other_user_prompts_checkbox
+                                        ],
+                                        [self.merged_meta_prompt_textbox])
+            
+            self.other_user_prompts_checkbox.change(self.update_enable_other_user_prompts,
+                                                    [self.other_user_prompts_checkbox],
+                                                    [
+                                                        self.other_user_prompts_textbox,
+                                                        self.meta_system_prompt_textbox
+                                                        ])
+
+
+        return prompt_ui
+
+    def update_enable_other_user_prompts(self, new_value):
+        self.enable_other_user_prompts = new_value
+        return \
+            gr.Textbox.update(visible=new_value), \
+                gr.Textbox.update(
+                    value = DEFAULT_META_SYSTEM_PROMPT_WITH_OTHER_PROMPTS if new_value else DEFAULT_META_SYSTEM_PROMPT
+                    )
+    
+    def merge_meta_system_prompt(
+            self, 
+            meta_system_prompt, 
+            current_system_prompt,
+            other_user_prompts,
+            testing_user_prompt, 
+            expect_output, 
+            current_output,
+            use_other_user_prompts
+            ):
+        """Merge meta and current system prompts."""
+
+        # converted_prompts = [prompt[0] for prompt in other_user_prompts.values]
+
+        user_prompt = self.generate_user_message(
+            current_system_prompt,
+            testing_user_prompt,
+            other_user_prompts if use_other_user_prompts else None,
+            expect_output, 
+            current_output
+        )
+
+        merged_prompt = f"{meta_system_prompt}\n\n{user_prompt}"
+
+        return merged_prompt
+
+    def copy_new_prompts(self, system_prompt, output):
+        """Copy prompts and output from new to current textboxes."""
+
+        return system_prompt, output
+
+    def test_prompt(
+            self, 
+            system_prompt, 
+            user_prompt,
+            model,
+            max_retries,
+            max_tokens,
+            request_timeout,
+            temperature,
+            ):
+        # Create the prompt
+        prompt = [
+            SystemMessage(content=system_prompt),
+            HumanMessage(content=user_prompt)
+        ]
+
+        chat_llm = ChatOpenAI(
+            model=model,
+            max_retries=max_retries,
+            max_tokens=None if max_tokens == 0 else max_tokens,
+            request_timeout=request_timeout,
+            temperature=temperature
+        )
+
+        # Get the response from OpenAI
+        gpt_response = chat_llm(prompt)
+
+        # Return the output to be placed in the output textbox
+        return gpt_response.content
+
+    def generate_user_message(self, current_system_prompt, testing_user_prompt, other_user_prompts, expect_output, current_output):
+        # other_prompts_formatted = '\n\n'.join([f"```\n{prompt}\n```" for prompt in other_user_prompts])
+        user_message = f"""
+* Prompt Template
+
+```
+{current_system_prompt}
+```
+
+* User Message
+
+```
+{testing_user_prompt}
+```
+
+* Other User Messages
+
+{other_user_prompts}
+
+* Expected GPT Message
+
+```
+{expect_output}
+```
+
+* GPT Message
+
+```
+{current_output}
+```
+""" if other_user_prompts is not None else f"""
+* Prompt Template
+
+```
+{current_system_prompt}
+```
+
+* User Message
+
+```
+{testing_user_prompt}
+```
+
+* Expected GPT Message
+
+```
+{expect_output}
+```
+
+* GPT Message
+
+```
+{current_output}
+```
+"""
+        return user_message
+
+    def meta_prompt(
+            self,
+            meta_system_prompt, 
+            current_system_prompt, 
+            testing_user_prompt, 
+            other_user_prompts,
+            expect_output, 
+            current_output,
+            use_user_prompts,
+            model,
+            max_retries,
+            max_tokens,
+            request_timeout,
+            temperature,
+            ):
+
+        # Format the user message
+        user_message = self.generate_user_message(
+            current_system_prompt,
+            testing_user_prompt,
+            other_user_prompts if use_user_prompts else None, 
+            expect_output, 
+            current_output
+        )
+
+        # Create the prompt
+        prompt = [
+            SystemMessage(content=meta_system_prompt),
+            HumanMessage(content=user_message)
+        ]
+
+        chat_llm = ChatOpenAI(
+            model=model,
+            max_retries=max_retries,
+            max_tokens=None if max_tokens == 0 else max_tokens,
+            request_timeout=request_timeout,
+            temperature=temperature
+        )
+
+        # Get the response from OpenAI
+        gpt_response = chat_llm(prompt)
+
+        updated_prompt = self.extract_updated_prompt(gpt_response.content)
+        changed = not self.detect_no_change(gpt_response.content)
+
+        # Return the output to be placed in the new system prompt textbox
+        if updated_prompt:
+            return updated_prompt, changed
+        else:
+            return gpt_response.content, changed
+
+    def extract_updated_prompt(self, gpt_response):
+        # Regular expression pattern to find the text enclosed
+        pattern = "<!-- BEGIN OF PROMPT -->(.*?)<!-- END OF PROMPT -->"
+        
+        # Using search method to find the first occurrence of the pattern
+        result = re.search(pattern, gpt_response, re.DOTALL)
+        
+        if result:
+            s = result.group(1).strip("\n")
+            if s.startswith("```") and s.endswith("```"):
+                s = s[3:-3]        
+            return s # Return the matched string
+        else:
+            return None  # If no such pattern is found return None
+
+    def detect_no_change(self, gpt_response):
+        # Regular expression pattern to find the exact string
+        pattern = "<!-- NO CHANGE TO PROMPT -->"
+        
+        # Using search method to find the occurrence of the pattern
+        result = re.search(pattern, gpt_response)
+        
+        if result:
+            return True  # If the pattern is found return True
+        else:
+            return False  # If no such pattern is found return False
+        
+    # def compare_strings(self, a: str, b: str, s: str) -> str:
+    #     # Create an instance of ChatOpenAI with the evaluation model
+    #     chat_model = OpenAI(temperature=0, model_name=self.llm_model_output_eval_dropdown.value)
+
+    #     # Create a prompt for comparison
+    #     prompt = (DEFAULT_OUTPUT_EVALUATING_PROMPT + 
+    #             '\n\n' + f'# S\n\n```\n{s}\n```\n\n# A\n\n```\n{a}\n```\n\n# B\n\n```\n{b}\n```\n\n')
+
+    #     # Get the response from OpenAI
+    #     response = chat_model(prompt)
+
+    #     # Remove '```' from beginning and end if it exists
+    #     if response.startswith("```") and response.endswith("```"):
+    #         response = response[3:-3]
+
+    #     # Check the first character of the response and return accordingly
+    #     if response.startswith('A'):
+    #         return 'A'
+    #     elif response.startswith('B'):
+    #         return 'B'
+    #     else:
+    #         return None
+
+    def optimize_prompt(
+            self, 
+            meta_system_prompt, 
+            current_system_prompt, 
+            testing_user_prompt,
+            other_user_prompts,
+            expect_output, 
+            current_output, 
+            iterations,
+            user_other_user_prompts,
+            meta_model,
+            meta_max_retries,
+            meta_max_tokens,
+            meta_request_timeout,
+            meta_temperature,
+            test_model,
+            test_max_retries,
+            test_max_tokens,
+            test_request_timeout,
+            test_temperature,
+            ):
+
+        changed = False
+
+        # Iterate the specified number of times
+        for i in range(int(iterations)):
+            # If current_output is None or not provided, get it from test_prompt
+            if current_output is None:
+                current_output = self.test_prompt(
+                    current_system_prompt, 
+                    testing_user_prompt,
+                    test_model,
+                    test_max_retries,
+                    test_max_tokens,
+                    test_request_timeout,
+                    test_temperature,
+                    )
+
+            # Call meta_prompt to get an optimized prompt
+            new_prompt, changed = self.meta_prompt(
+                meta_system_prompt, 
+                current_system_prompt, 
+                testing_user_prompt,
+                other_user_prompts,
+                expect_output, 
+                current_output,
+                user_other_user_prompts,
+                meta_model,
+                meta_max_retries,
+                meta_max_tokens,
+                meta_request_timeout,
+                meta_temperature,
+                )
+
+            # If changed is False, break the loop
+            if not changed:
+                break
+
+            # If there is an updated prompt and it's different from the current one, update current_system_prompt
+            if new_prompt and new_prompt != current_system_prompt:
+                current_system_prompt = new_prompt
+                # Reset current_output to None so it gets recalculated in the next iteration
+                current_output = None
+
+        return current_system_prompt, changed  # Return the optimized system prompt
+
+    def compare_strings(self, alpha: str, beta: str, expected: str) -> str:
+        # If both ALPHA and BETA are empty, return None
+        if not alpha and not beta:
+            return None
+
+        # If either ALPHA or BETA is empty, the non-empty string should be considered more similar to EXPECTED
+        if not alpha:
+            return 'B'
+        if not beta:
+            return 'A'
+
+        # If both ALPHA and BETA are identical, return None
+        if alpha == beta:
+            return None
+
+        # Create the CountVectorizer instance
+        vectorizer = CountVectorizer().fit_transform([alpha, beta, expected])
+        vectors = vectorizer.toarray()
+
+        # Calculate cosine similarities
+        alpha_sim = cosine_similarity(vectors[0].reshape(1, -1), vectors[2].reshape(1, -1))
+        beta_sim = cosine_similarity(vectors[1].reshape(1, -1), vectors[2].reshape(1, -1))
+
+        # Compare similarities and return the string that is more similar to the expected string
+        if alpha_sim > beta_sim:
+            return 'A'
+        elif beta_sim > alpha_sim:
+            return 'B'
+        else:
+            return None
+        
+    def delta_similarities(self, alpha: str, beta: str, expected: str) -> float:
+        # If both ALPHA and BETA are empty, return 0
+        if not alpha and not beta:
+            return 0.0
+
+        # If either ALPHA or BETA is empty, the non-empty string should be considered more similar to EXPECTED
+        if not alpha:
+            return -1.0
+        if not beta:
+            return 1.0
+
+        # If both ALPHA and BETA are identical, return 0
+        if alpha == beta:
+            return 0.0
+
+        # Create the CountVectorizer instance
+        vectorizer = CountVectorizer().fit_transform([alpha, beta, expected])
+        vectors = vectorizer.toarray()
+
+        # Calculate cosine similarities
+        alpha_sim = cosine_similarity(vectors[0].reshape(1, -1), vectors[2].reshape(1, -1))
+        beta_sim = cosine_similarity(vectors[1].reshape(1, -1), vectors[2].reshape(1, -1))
+
+        # Return the difference in similarities
+        return alpha_sim[0][0] - beta_sim[0][0]
+
+    def compare_outputs(self, new_output, current_output, expected_output):
+        # Compare new output and current output against expected output
+        # result = self.compare_strings(new_output, current_output, expected_output)
+        result = self.delta_similarities(new_output, current_output, expected_output)
+
+        return result

meta_prompt/__init__.py

@@ -0,0 +1,4 @@
+__version__ = '0.1.0'
+
+from .meta_prompt import AgentState, MetaPromptGraph
+

meta_prompt/meta_prompt.py

@@ -0,0 +1,442 @@
+import typing
+import pprint
+import logging
+from typing import Dict, Any, Callable, List, Union, Optional
+from langchain_core.language_models import BaseLanguageModel
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.prompts import ChatPromptTemplate
+from langgraph.graph import StateGraph, END
+from langgraph.checkpoint.memory import MemorySaver
+from langgraph.errors import GraphRecursionError
+from pydantic import BaseModel
+
+class AgentState(BaseModel):
+    max_output_age: int = 0
+    user_message: Optional[str] = None
+    expected_output: Optional[str] = None
+    acceptance_criteria: Optional[str] = None
+    system_message: Optional[str] = None
+    output: Optional[str] = None
+    suggestions: Optional[str] = None
+    accepted: bool = False
+    analysis: Optional[str] = None
+    best_output: Optional[str] = None
+    best_system_message: Optional[str] = None
+    best_output_age: int = 0
+
+class MetaPromptGraph:
+    NODE_PROMPT_INITIAL_DEVELOPER = "prompt_initial_developer"
+    NODE_PROMPT_DEVELOPER = "prompt_developer"
+    NODE_PROMPT_EXECUTOR = "prompt_executor"
+    NODE_OUTPUT_HISTORY_ANALYZER = "output_history_analyzer"
+    NODE_PROMPT_ANALYZER = "prompt_analyzer"
+    NODE_PROMPT_SUGGESTER = "prompt_suggester"
+
+    DEFAULT_PROMPT_TEMPLATES = {
+        NODE_PROMPT_INITIAL_DEVELOPER: ChatPromptTemplate.from_messages([
+            ("system", """# Expert Prompt Engineer
+
+You are an expert prompt engineer tasked with creating system messages for AI
+assistants.
+
+## Instructions
+
+1. Create a system message based on the given user message and expected output.
+2. Ensure the system message can handle similar user messages.
+3. Output only the system message, without any additional content.
+4. Expected Output text should not appear in System Message as an example. But
+   it's OK to use some similar text as an example instead.
+5. Format the system message well, with no more than 80 characters per line
+   (except for raw text).
+
+## Output
+
+Provide only the system message, adhering to the above guidelines.
+"""),
+            ("human", """# User Message
+             
+{user_message}
+
+# Expected Output
+
+{expected_output}
+""")
+        ]),
+        NODE_PROMPT_DEVELOPER: ChatPromptTemplate.from_messages([
+            ("system", """# Expert Prompt Engineer
+
+You are an expert prompt engineer tasked with updating system messages for AI
+assistants. You Update System Message according to Suggestions, to improve
+Output and match Expected Output more closely.
+
+## Instructions
+
+1. Update the system message based on the given Suggestion, User Message, and
+   Expected Output.
+2. Ensure the updated system message can handle similar user messages.
+3. Modify only the content mentioned in the Suggestion. Do not change the
+   parts that are not related to the Suggestion.
+4. Output only the updated system message, without any additional content.
+5. Expected Output text should not appear in System Message as an example. But
+   it's OK to use some similar text as an example instead.
+   * Remove the Expected Output text or text highly similar to Expected Output
+     from System Message, if it's present.
+6. Format the system message well, with no more than 80 characters per line
+   (except for raw text).
+
+## Output
+
+Provide only the updated System Message, adhering to the above guidelines.
+"""),
+            ("human", """# Current system message
+
+{system_message}
+
+# User Message
+
+{user_message}
+
+# Expected Output
+
+{expected_output}
+
+# Suggestions
+
+{suggestions}
+""")
+        ]),
+        NODE_PROMPT_EXECUTOR: ChatPromptTemplate.from_messages([
+            ("system", "{system_message}"),
+            ("human", "{user_message}")
+        ]),
+        NODE_OUTPUT_HISTORY_ANALYZER: ChatPromptTemplate.from_messages([
+            ("system", """You are a text comparing program. You read the Acceptance Criteria, compare the
+compare the exptected output with two different outputs, and decide which one is
+more similar to the expected output.
+
+You output the following analysis according to the Acceptance Criteria:
+
+* Your analysis in a Markdown list.
+* The ID of the output that is more similar to the Expected Output as Preferred
+    Output ID, with the following format:
+    
+```
+# Analysis
+
+...
+
+# Preferred Output ID: [ID]
+```
+
+If both outputs are equally similar to the expected output, output the following:
+
+```
+# Analysis
+
+...
+
+# Draw
+```
+"""),
+            ("human", """
+# Output ID: A
+
+```
+{best_output}
+```
+
+# Output ID: B
+
+```
+{output}
+```
+
+# Acceptance Criteria
+
+{acceptance_criteria}
+
+# Expected Output
+
+```
+{expected_output}
+```
+""")
+        ]),
+        NODE_PROMPT_ANALYZER: ChatPromptTemplate.from_messages([
+            ("system", """
+You are a text comparing program. You compare the following output texts,
+analysis the System Message and provide a detailed analysis according to
+`Acceptance Criteria`. Then you decide whether `Actual Output` is acceptable.
+
+Provide your analysis in the following format:
+
+```
+- Acceptable Differences: [List acceptable differences succinctly]
+- Unacceptable Differences: [List unacceptable differences succinctly]
+- Accept: [Yes/No]
+```
+
+* Compare Expected Output and Actual Output with the guidance of Accept Criteria.
+* Only set 'Accept' to 'Yes', if Accept Criteria are all met. Otherwise, set 'Accept' to 'No'.
+* List only the acceptable differences according to Accept Criteria in 'acceptable Differences' section.
+* List only the unacceptable differences according to Accept Criteria in 'Unacceptable Differences' section.
+
+# Acceptance Criteria
+
+```
+{acceptance_criteria}
+```
+"""),
+            ("human", """
+# System Message
+
+```
+{system_message}
+```
+
+# Expected Output
+
+```
+{expected_output}
+```
+
+# Actual Output
+
+```
+{output}
+```
+""")
+        ]),
+        NODE_PROMPT_SUGGESTER: ChatPromptTemplate.from_messages([
+            ("system", """
+Read the following inputs and outputs of an LLM prompt, and also analysis about them.
+Then suggest how to improve System Prompt.
+
+System Prompt:
+```
+{system_message}
+```
+User Message:
+```
+{user_message}
+```
+Expected Output: 
+```
+{expected_output}
+```
+Actual Output: 
+```
+{output}
+```
+
+Acceptance Criteria:
+```
+{acceptance_criteria}
+```
+
+Analysis:
+```
+{analysis}
+```
+
+* The goal is to improve the System Prompt to match the Expected Output better.
+* Ignore all Acceptable Differences and focus on Unacceptable Differences.
+* Suggest formal changes first, then semantic changes.
+* Provide your suggestions in a Markdown list, nothing else. Output only the
+    suggestions related with Unacceptable Differences.
+    * Use `... should ...` to clearly state the desired output.
+    * Figue out the contexts of the System Message that conflict with the suggestions,
+    and suggest modification or deletion.
+* Expected Output text should not appear in System Message as an example. But
+    it's OK to use some similar text as an example instead.
+    * Ask to remove the Expected Output text or text highly similar to Expected Output
+    from System Message, if it's present.
+* Provide format examples or detected format name, if System Message does not.
+    * Specify the detected format name (e.g. XML, JSON, etc.) of Expected Output, if
+    System Message does not mention it.
+""")
+        ])
+    }
+
+    @classmethod
+    def get_node_names(cls):
+        return [
+            cls.NODE_PROMPT_INITIAL_DEVELOPER,
+            cls.NODE_PROMPT_DEVELOPER,
+            cls.NODE_PROMPT_EXECUTOR,
+            cls.NODE_OUTPUT_HISTORY_ANALYZER,
+            cls.NODE_PROMPT_ANALYZER,
+            cls.NODE_PROMPT_SUGGESTER
+        ]
+
+    def __init__(self,
+                 llms: Union[BaseLanguageModel, Dict[str, BaseLanguageModel]] = {},
+                 prompts: Dict[str, ChatPromptTemplate] = {},
+                 verbose = False):
+        self.logger = logging.getLogger(__name__)
+        if verbose:
+            self.logger.setLevel(logging.DEBUG)
+        else:
+            self.logger.setLevel(logging.INFO)
+
+        if isinstance(llms, BaseLanguageModel):
+            self.llms: Dict[str, BaseLanguageModel] = {node: llms for node in self.get_node_names()}
+        else:
+            self.llms: Dict[str, BaseLanguageModel] = llms
+        self.prompt_templates: Dict[str, ChatPromptTemplate] = self.DEFAULT_PROMPT_TEMPLATES.copy()
+        self.prompt_templates.update(prompts)
+
+    def _create_workflow(self, including_initial_developer: bool = True) -> StateGraph:
+        workflow = StateGraph(AgentState)
+            
+        workflow.add_node(self.NODE_PROMPT_DEVELOPER,
+                          lambda x: self._prompt_node(
+                              self.NODE_PROMPT_DEVELOPER,
+                              "system_message",
+                              x))
+        workflow.add_node(self.NODE_PROMPT_EXECUTOR,
+                          lambda x: self._prompt_node(
+                              self.NODE_PROMPT_EXECUTOR,
+                              "output",
+                              x))
+        workflow.add_node(self.NODE_OUTPUT_HISTORY_ANALYZER,
+                          lambda x: self._output_history_analyzer(x))
+        workflow.add_node(self.NODE_PROMPT_ANALYZER,
+                          lambda x: self._prompt_analyzer(x))
+        workflow.add_node(self.NODE_PROMPT_SUGGESTER,
+                          lambda x: self._prompt_node(
+                              self.NODE_PROMPT_SUGGESTER,
+                              "suggestions",
+                              x))
+
+        workflow.add_edge(self.NODE_PROMPT_DEVELOPER, self.NODE_PROMPT_EXECUTOR)
+        workflow.add_edge(self.NODE_PROMPT_EXECUTOR, self.NODE_OUTPUT_HISTORY_ANALYZER)
+        workflow.add_edge(self.NODE_PROMPT_SUGGESTER, self.NODE_PROMPT_DEVELOPER)
+
+        workflow.add_conditional_edges(
+            self.NODE_OUTPUT_HISTORY_ANALYZER,
+            lambda x: self._should_exit_on_max_age(x),
+            {
+                "continue": self.NODE_PROMPT_ANALYZER,
+                "rerun": self.NODE_PROMPT_SUGGESTER,
+                END: END
+            }
+        )
+
+        workflow.add_conditional_edges(
+            self.NODE_PROMPT_ANALYZER,
+            lambda x: self._should_exit_on_acceptable_output(x),
+            {
+                "continue": self.NODE_PROMPT_SUGGESTER,
+                END: END
+            }
+        )
+
+        if including_initial_developer:
+            workflow.add_node(self.NODE_PROMPT_INITIAL_DEVELOPER,
+                            lambda x: self._prompt_node(
+                                self.NODE_PROMPT_INITIAL_DEVELOPER,
+                                "system_message",
+                                x))
+            workflow.add_edge(self.NODE_PROMPT_INITIAL_DEVELOPER, self.NODE_PROMPT_EXECUTOR)
+            workflow.set_entry_point(self.NODE_PROMPT_INITIAL_DEVELOPER)
+        else:
+            workflow.set_entry_point(self.NODE_PROMPT_EXECUTOR)
+
+        return workflow
+
+    def __call__(self, state: AgentState, recursion_limit: int = 25) -> AgentState:
+        workflow = self._create_workflow(including_initial_developer=(state.system_message is None or state.system_message == ""))
+
+        memory = MemorySaver()
+        graph = workflow.compile(checkpointer=memory)
+        config = {"configurable": {"thread_id": "1"}, "recursion_limit": recursion_limit}
+
+        try:
+            self.logger.debug("Invoking graph with state: %s", pprint.pformat(state))
+
+            output_state = graph.invoke(state, config)
+
+            self.logger.debug("Output state: %s", pprint.pformat(output_state))
+
+            return output_state
+        except GraphRecursionError as e:
+            self.logger.info("Recursion limit reached. Returning the best state found so far.")
+            checkpoint_states = graph.get_state(config)
+
+            # if the length of states is bigger than 0, print the best system message and output
+            if len(checkpoint_states) > 0:
+                output_state = checkpoint_states[0]
+                return output_state
+            else:
+                self.logger.info("No checkpoint states found. Returning the input state.")
+            
+        return state
+
+    def _prompt_node(self, node, target_attribute: str, state: AgentState) -> AgentState:
+        prompt = self.prompt_templates[node].format_messages(**state.model_dump())
+
+        self.logger.debug("Invoking %s with prompt: %s", node, pprint.pformat(prompt))
+        response = self.llms[node].invoke(self.prompt_templates[node].format_messages(**state.model_dump()))
+        self.logger.debug("Response: %s", response.content)
+        
+        setattr(state, target_attribute, response.content)
+        return state
+
+    def _output_history_analyzer(self, state: AgentState) -> AgentState:
+        if state.best_output is None:
+            state.best_output = state.output
+            state.best_system_message = state.system_message
+            state.best_output_age = 0
+
+            self.logger.debug("Best output initialized to the current output: \n %s", state.output)
+
+            return state
+
+        prompt = self.prompt_templates[self.NODE_OUTPUT_HISTORY_ANALYZER].format_messages(**state.model_dump())
+
+        self.logger.debug("Invoking %s with prompt: %s",
+                          self.NODE_OUTPUT_HISTORY_ANALYZER, 
+                          pprint.pformat(prompt))
+        response = self.llms[self.NODE_OUTPUT_HISTORY_ANALYZER].invoke(prompt)
+        self.logger.debug("Response: %s", response.content)
+
+        analysis = response.content
+
+        if state.best_output is None or "# Preferred Output ID: B" in analysis:
+            state.best_output = state.output
+            state.best_system_message = state.system_message
+            state.best_output_age = 0
+
+            self.logger.debug("Best output updated to the current output: \n %s", state.output)
+        else:
+            state.best_output_age += 1
+
+            self.logger.debug("Best output age incremented to %s", state.best_output_age)
+
+        return state
+
+    def _prompt_analyzer(self, state: AgentState) -> AgentState:
+        prompt = self.prompt_templates[self.NODE_PROMPT_ANALYZER].format_messages(**state.model_dump())
+
+        self.logger.debug("Invoking %s with prompt: %s",
+                          self.NODE_PROMPT_ANALYZER,
+                          pprint.pformat(prompt))
+        response = self.llms[self.NODE_PROMPT_ANALYZER].invoke(prompt)
+        self.logger.debug("Response: %s", response.content)
+
+        state.analysis = response.content
+        state.accepted = "Accept: Yes" in response.content
+
+        self.logger.debug("Accepted: %s", state.accepted)
+
+        return state
+
+    def _should_exit_on_max_age(self, state: AgentState) -> str:
+        if state.max_output_age <= 0 or state.best_output_age < state.max_output_age:
+            return "continue"
+        else:
+            return "rerun"
+
+    def _should_exit_on_acceptable_output(self, state: AgentState) -> str:
+        return "continue" if not state.accepted else END

pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "meta-prompt"
+version = "0.1.0"
+description = "This package generates high quality prompts with input and expected output examples."
+authors = ["Yale Huang <[email protected]>"]
+license = "MIT"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+langgraph = "^0.1.5"
+langchain = "^0.2.6"
+langchain-openai = "^0.1.14"
+pydantic = "^2.8.2"
+
+[tool.poetry.dev-dependencies]
+gradio = "^4.37.2"
+confz = "^2.0.1"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.2.2"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,122 @@
+aiofiles==23.2.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.1.1
+annotated-types==0.5.0
+anyio==3.7.1
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.1.0
+certifi==2023.7.22
+charset-normalizer==3.2.0
+click==8.1.7
+comm==0.2.2
+confz==2.0.1
+contourpy==1.1.1
+cycler==0.11.0
+dataclasses-json==0.6.0
+debugpy==1.8.2
+decorator==5.1.1
+distro==1.9.0
+exceptiongroup==1.2.1
+executing==2.0.1
+fastapi==0.103.1
+ffmpy==0.3.1
+filelock==3.12.4
+fonttools==4.42.1
+frozenlist==1.4.0
+fsspec==2023.9.2
+gradio==4.37.2
+gradio_client==1.0.2
+greenlet==2.0.2
+h11==0.14.0
+httpcore==0.18.0
+httpx==0.25.0
+huggingface-hub==0.23.4
+idna==3.4
+importlib-resources==6.1.0
+ipykernel==6.29.4
+ipython==8.26.0
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+kiwisolver==1.4.5
+langchain==0.2.6
+langchain-core==0.2.10
+langchain-openai==0.1.13
+langchain-text-splitters==0.2.2
+langgraph==0.1.4
+langsmith==0.1.82
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.0
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+multidict==6.0.4
+mypy-extensions==1.0.0
+nest-asyncio==1.6.0
+numexpr==2.8.6
+numpy==1.26.0
+openai==1.35.7
+orjson==3.10.5
+packaging==24.1
+pandas==2.1.1
+parso==0.8.4
+pexpect==4.9.0
+Pillow==10.0.1
+platformdirs==4.2.2
+prompt_toolkit==3.0.47
+psutil==6.0.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pydantic==2.3.0
+pydantic_core==2.6.3
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2023.3.post1
+PyYAML==6.0.1
+pyzmq==26.0.3
+referencing==0.30.2
+regex==2024.5.15
+requests==2.31.0
+rich==13.7.1
+rpds-py==0.10.3
+ruff==0.5.0
+scikit-learn==1.3.1
+scipy==1.11.3
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.21
+stack-data==0.6.3
+starlette==0.27.0
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tiktoken==0.7.0
+toml==0.10.2
+tomlkit==0.12.0
+toolz==0.12.0
+tornado==6.4.1
+tqdm==4.66.1
+traitlets==5.14.3
+typer==0.12.3
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+tzdata==2023.3
+urllib3==2.0.5
+uvicorn==0.23.2
+wcwidth==0.2.13
+websockets==11.0.3
+yarl==1.9.2

tests/meta_prompt_graph_test.py

@@ -0,0 +1,160 @@
+import unittest
+import pprint
+import logging
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+# Assuming the necessary imports are made for the classes and functions used in meta_prompt_graph.py
+from meta_prompt import AgentState, MetaPromptGraph
+
+from langchain_openai import ChatOpenAI
+
+class TestMetaPromptGraph(unittest.TestCase):
+    def setUp(self):
+        # logging.basicConfig(level=logging.DEBUG)
+        pass
+
+    def test_prompt_node(self):
+        llms = {
+            MetaPromptGraph.NODE_PROMPT_INITIAL_DEVELOPER: MagicMock(
+                invoke=MagicMock(return_value=MagicMock(content="Mocked response content"))
+            )
+        }
+
+        # Create an instance of MetaPromptGraph with the mocked language model and template
+        graph = MetaPromptGraph(llms=llms)
+
+        # Create a mock AgentState
+        state = AgentState(user_message="Test message", expected_output="Expected output")
+
+        # Invoke the _prompt_node method with the mock node, target attribute, and state
+        updated_state = graph._prompt_node(
+            MetaPromptGraph.NODE_PROMPT_INITIAL_DEVELOPER, "output", state
+        )
+
+        # Assertions
+        assert updated_state.output == "Mocked response content", \
+            "The output attribute should be updated with the mocked response content"
+
+    def test_output_history_analyzer(self):
+        # Setup
+        llms = {
+            "output_history_analyzer": MagicMock(invoke=lambda prompt: MagicMock(content="""# Analysis
+
+    This analysis compares two outputs to the expected output based on specific criteria.
+
+    # Preferred Output ID: B"""))
+        }
+        prompts = {}
+        meta_prompt_graph = MetaPromptGraph(llms=llms, prompts=prompts)
+        state = AgentState(
+            user_message="How do I reverse a list in Python?",
+            expected_output="Use the `[::-1]` slicing technique or the `list.reverse()` method.",
+            output="To reverse a list in Python, you can use the `[::-1]` slicing.",
+            system_message="To reverse a list, use slicing or the reverse method.",
+            best_output="To reverse a list in Python, use the `reverse()` method.",
+            best_system_message="To reverse a list, use the `reverse()` method.",
+            acceptance_criteria="The output should correctly describe how to reverse a list in Python."
+        )
+
+        # Invoke the output history analyzer node
+        updated_state = meta_prompt_graph._output_history_analyzer(state)
+
+        # Assertions
+        assert updated_state.best_output == state.output, \
+            "Best output should be updated to the current output."
+        assert updated_state.best_system_message == state.system_message, \
+            "Best system message should be updated to the current system message."
+        assert updated_state.best_output_age == 0, \
+            "Best output age should be reset to 0."
+
+    def test_prompt_analyzer_accept(self):
+        llms = {
+            MetaPromptGraph.NODE_PROMPT_ANALYZER: MagicMock(
+                invoke=lambda prompt: MagicMock(content="Accept: Yes"))
+        }
+        meta_prompt_graph = MetaPromptGraph(llms)
+        state = AgentState(output="Test output", expected_output="Expected output")
+        updated_state = meta_prompt_graph._prompt_analyzer(state)
+        assert updated_state.accepted == True
+
+    def test_workflow_execution(self):
+        # MODEL_NAME = "anthropic/claude-3.5-sonnet:beta"
+        # MODEL_NAME = "meta-llama/llama-3-70b-instruct"
+        MODEL_NAME = "deepseek/deepseek-chat"
+        # MODEL_NAME = "google/gemma-2-9b-it"
+        # MODEL_NAME = "recursal/eagle-7b"
+        # MODEL_NAME = "meta-llama/llama-3-8b-instruct"
+        llm = ChatOpenAI(model_name=MODEL_NAME)
+
+        meta_prompt_graph = MetaPromptGraph(llms=llm)
+        input_state = AgentState(
+            user_message="How do I reverse a list in Python?",
+            expected_output="Use the `[::-1]` slicing technique or the `list.reverse()` method.",
+            acceptance_criteria="Similar in meaning, text length and style."
+            )
+        output_state = meta_prompt_graph(input_state, recursion_limit=25)
+
+        pprint.pp(output_state)
+        # if output_state has key 'best_system_message', print it
+        assert 'best_system_message' in output_state, \
+            "The output state should contain the key 'best_system_message'"
+        assert output_state['best_system_message'] is not None, \
+            "The best system message should not be None"
+        if 'best_system_message' in output_state and output_state['best_system_message'] is not None:
+            print(output_state['best_system_message'])
+
+        # try another similar user message with the generated system message
+        user_message = "How can I create a list of numbers in Python?"
+        messages = [("system", output_state['best_system_message']), 
+                    ("human", user_message)]
+        result = llm.invoke(messages)
+
+        # assert attr 'content' in result
+        assert hasattr(result, 'content'), \
+            "The result should have the attribute 'content'"
+        print(result.content)
+
+    def test_workflow_execution_with_llms(self):
+        optimizer_llm = ChatOpenAI(model_name="deepseek/deepseek-chat", temperature=0.5)
+        executor_llm = ChatOpenAI(model_name="meta-llama/llama-3-8b-instruct", temperature=0.01)
+
+        llms = {
+            MetaPromptGraph.NODE_PROMPT_INITIAL_DEVELOPER: optimizer_llm,
+            MetaPromptGraph.NODE_PROMPT_DEVELOPER: optimizer_llm,
+            MetaPromptGraph.NODE_PROMPT_EXECUTOR: executor_llm,
+            MetaPromptGraph.NODE_OUTPUT_HISTORY_ANALYZER: optimizer_llm,
+            MetaPromptGraph.NODE_PROMPT_ANALYZER: optimizer_llm,
+            MetaPromptGraph.NODE_PROMPT_SUGGESTER: optimizer_llm
+        }
+
+        meta_prompt_graph = MetaPromptGraph(llms=llms)
+        input_state = AgentState(
+            user_message="How do I reverse a list in Python?",
+            expected_output="Use the `[::-1]` slicing technique or the `list.reverse()` method.",
+            acceptance_criteria="Similar in meaning, text length and style."
+            )
+        output_state = meta_prompt_graph(input_state, recursion_limit=25)
+
+        pprint.pp(output_state)
+        # if output_state has key 'best_system_message', print it
+        assert 'best_system_message' in output_state, \
+            "The output state should contain the key 'best_system_message'"
+        assert output_state['best_system_message'] is not None, \
+            "The best system message should not be None"
+        if 'best_system_message' in output_state and output_state['best_system_message'] is not None:
+            print(output_state['best_system_message'])
+
+        # try another similar user message with the generated system message
+        user_message = "How can I create a list of numbers in Python?"
+        messages = [("system", output_state['best_system_message']), 
+                    ("human", user_message)]
+        result = executor_llm.invoke(messages)
+
+        # assert attr 'content' in result
+        assert hasattr(result, 'content'), \
+            "The result should have the attribute 'content'"
+        print(result.content)
+
+if __name__ == '__main__':
+    unittest.main()