Prepare for GitHub.

.gitignore

@@ -0,0 +1,3 @@
+.venv
+.vscode
+__pycache__

README

@@ -0,0 +1,40 @@
+# Meta Prompt Demo
+
+This project is a demonstration of the concept of Meta Prompt, which involves generating a language model prompt using another language model. The demo showcases how a language model can be used to generate high-quality prompts for another language model.
+
+## Overview
+
+The demo utilizes OpenAI's language models and provides a user interface for interacting with the chatbot. It allows users to input prompts, execute model calls, compare outputs, and optimize prompts based on desired criteria.
+
+## Installation
+
+To use this demo, please follow these steps:
+
+1. Clone the repository: `git clone https://github.com/your-username/meta-prompt-demo.git`
+2. Change into the project directory: `cd meta-prompt-demo`
+3. Install the required dependencies: `pip install -r requirements.txt`
+
+Please note that you need to have Python and pip installed on your system.
+
+## Usage
+
+To run the demo, execute the following command:
+
+```
+python main.py --api_key YOUR_API_KEY
+```
+
+Replace `YOUR_API_KEY` with your OpenAI API key. Other optional parameters can be specified as well, such as proxy settings, model name, API base URL, maximum message length, sharing option, and advanced mode. Please refer to the command-line argument options in the script for more details.
+
+Once the demo is running, you can interact with the chatbot through the user interface provided. Enter prompts, execute model calls, compare outputs, and explore the functionality of the Meta Prompt concept.
+
+## License
+
+This project is licensed under the MIT License. Please see the [LICENSE](LICENSE) file for more information.
+
+## Contact
+
+For any questions or feedback regarding this project, please feel free to reach out to Yale Huang at [email protected].
+
+---
+*Note: This README serves as a general guideline for setting up and using the Meta Prompt demo. Make sure to update it with more specific information if necessary.*

meta_prompt.py

@@ -0,0 +1,54 @@
+"""
+MIT License
+
+Copyright (c) 2023 Yale Huang
+Email: [email protected]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+"""
+
+import argparse
+import os
+import openai
+
+import gradio as gr
+from prompt_ui import PromptUI
+
+class ChatbotApp:
+    def __init__(self, args):
+        os.environ["OPENAI_API_KEY"] = args.api_key
+        if args.proxy:
+            openai.proxy = eval(args.proxy)
+
+        self.prompt_ui = PromptUI(advanced_mode=args.advanced_mode)
+
+        self.ui = gr.TabbedInterface(
+            [self.prompt_ui.ui], 
+            ['Prompt']
+        )
+    def launch(self, *args, **kwargs):
+        self.ui.launch(args, kwargs)
+    
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--api_key", type=str, required=True, help="OpenAI API key")
+    parser.add_argument("--proxy", type=str, default=None, help="Proxy settings")
+    parser.add_argument("--share", action='store_true',
+                        help="Launch app with sharing option")
+    parser.add_argument("--advanced_mode", action='store_true', default=False,
+                        help="Enable advanced mode")
+ 
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    args = parse_args()
+    app = ChatbotApp(args)
+    app.ui.queue().launch(share=args.share)

prompt_ui.py

@@ -0,0 +1,736 @@
+"""
+MIT License
+
+Copyright (c) 2023 Yale Huang
+Email: [email protected]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+"""
+
+import re
+import gradio as gr
+
+from langchain.chat_models import ChatOpenAI
+from langchain.llms.openai import OpenAI
+from langchain.schema import AIMessage, HumanMessage, SystemMessage
+from langchain.prompts import SystemMessagePromptTemplate, HumanMessagePromptTemplate
+from langchain.prompts.chat import ChatPromptTemplate
+
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+gpt_models_not_legacy = [
+    "gpt-4",
+    "gpt-4-0613",
+    "gpt-4-32k",
+    "gpt-4-32k-0613",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-16k",
+    "gpt-3.5-turbo-instruct",
+    "gpt-3.5-turbo-0613"
+]
+
+gpt_models_legacy = [
+    "gpt-3.5-turbo-instruct",
+    "gpt-3.5-turbo-0301",
+    "text-davinci-003",
+    "text-davinci-002",
+    "code-davinci-002"
+]
+
+DEFAULT_MODEL_FOR_GENERATING="gpt-4"
+DEFAULT_MODEL_FOR_TESTING="gpt-3.5-turbo"
+DEFAULT_MODEL_FOR_OUTPUT_EVALUATING="gpt-3.5-turbo-instruct"
+
+DEFAULT_META_SYSTEM_PROMPT = \
+'''
+You are a Prompt Engineer. You review the Prompt template for GTP-3.5 and suggest changes.
+
+# Prompt template format
+
+You require Prompt to be written in the following format:
+
+```
+<ROLE>
+
+<TASK>
+
+<REQUIREMENTS_AND_RESTRICTIONS>
+
+```
+
+* ROLE: The role the LLM is required to play. Describe it in one sentence.
+* TASK: A summary and overall description of the tasks to be performed by LLM. Describe it in one or more sentences.
+* REQUIREMENTS_AND_RESTRICTIONS: Specific requirements for the task. Describe using Markdown List.
+
+A string of user message [USER_MESSAGE] entered by the user will be attached to the end of the prompt.
+
+# Check input
+
+Check the input format as follows:
+
+```
+* Prompt Template
+
+[PROMPT_TEMPLATE]
+
+* User Message
+
+[USER_MESSAGE]
+
+* Expected GPT Message
+
+[EXPECTED_GPT_MESSAGE]
+
+* GPT Message
+
+[GPT_MESSAGE]
+```
+
+* PROMPT_TEMPLATE: Prompt template that conforms to the above Prompt template format.
+* USER_MESSAGE: User input. Used to replace {user_message} in the Prompt template.
+* EXPECTED_GPT_MESSAGE: Expect output generated by GPT.
+* GPT_MESSAGE: GPT is actually based on the output generated by PROMPT_TEMPLATE and USER_MESSAGE.
+
+# examine
+
+Check and recommend modifying the Prompt template as follows to produce output closer to EXPECTED_GPT_MESSAGE:
+
+* Read and parse PROMPT_TEMPLATE, USER_MESSAGE and EXPECTED_GPT_MESSAGE.
+   * Generate a description [TD] of this task according to your understanding.
+   * Analyze the correlation between PROMPT_TEMPLATE and USER_MESSAGE [UMR].
+   * Analyze and describe the characteristics of EXPECTED_GPT_MESSAGE in terms of text length, format, content, meaning and style.
+   * Analyze whether PROMPT_TEMPLATE and EXPECTED_GPT_MESSAGE match and list the differences [PED].
+* Check whether GPT_MESSAGE conforms to EXPECTED_GPT_MESSAGE. Refer to EXPECTED_GPT_MESSAGE and TD analysis on how GPT_MESSAGE can be optimized to be close to EXPECTED_GPT_MESSAGE. Modification suggestions are listed in detail [MCSL].
+   * Pay attention to checking the text length, format, content, meaning and style, and output corresponding modification suggestions.
+     * Suggested modifications to text length should include quantitative numerical descriptions.
+     * Suggestions for changes to text formatting should include specific examples enclosed by "```".
+   * Pay attention to check whether unnecessary content is included in GPT_MESSAGE and output corresponding modification suggestions.
+   * Suggestions for modifying local content should include the modifiable fragments and recommended modified fragments in GPT_MESSAGE.
+* Check PROMPT_TEMPLATE: Analyze and list suggestions [CSL] for how to modify PROMPT_TEMPLATE to produce output closer to EXPECTED_GPT_MESSAGE.
+   * For requirements that have been stated in REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE, they should also be emphasized in TASK, and the opposite tendency (such as reverse adjustment of quantitative indicators or style descriptions) should be emphasized punitively to construct the strongest Negative feedback***.
+    * For format requirements that have been stated in REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE, add an example enclosed with "```".
+   * Based on PED recommendations on how to modify PROMPT_TEMPLATE.
+   * Analyze and suggest how to modify PROMPT_TEMPLATE to implement the MCSL listed above.
+   * Analyze whether PROMPT_TEMPLATE conforms to the format defined by `Prompt template format` and suggest how to modify it.
+   * Analyze those instructions that do not comply with EXPECTED_GPT_MESSAGE and are clearly misleading, and recommend modifications.
+   * Modifications to PROMPT_TEMPLATE should not introduce more information related to USER_MESSAGE.
+   * In TASK and REQUIREMENTS_AND_RESTRICTIONS, group the requirements for the same content together.
+   * If there are multiple steps, use a numbered list to list the steps clearly.
+   * Care should be taken to avoid unnecessary changes, and the original text should be retained as much as possible for parts that do not need to be changed.
+   * Only output [CSL], do not output the modified PROMPT_TEMPLATE.
+* Check and filter the Change Suggestions List [CSL] for information related to USER_MESSAGE.
+   * Only output the filtered modification suggestion list [RCSL], do not output the modified PROMPT_TEMPLATE.
+* Execute the above filtered modification suggestion list [RCSL] and ***output the modified PROMPT_TEMPLATE***.
+  * Execute RCSL only, avoid other changes.
+  * Care should be taken to avoid unnecessary changes, and the original text should be retained as much as possible for parts that do not need to be changed, except the requirements that have been stated in TASK or REQUIREMENTS_AND_RESTRICTIONS but are not met by GPT_MESSAGE.
+  * Strictly use the following format for output:
+```
+<!-- BEGIN OF PROMPT -->
+
+<Updated Prompt>
+
+<!-- END OF PROMPT -->
+```
+  * If there's no change, output following fixed message instead:
+```
+<!-- NO CHANGE TO PROMPT -->
+```
+* Evaluation modified PROMPT_TEMPLATE.
+   * Analyze the changes it may cause in the output of LLM [EC].
+   * Analyze whether EC would be more consistent with EXPECTED_GPT_MESSAGE.
+   * Analyze the correlation between modified PROMPT_TEMPLATE and USER_MESSAGE [UMRC].
+   * Analyze UMR and UMRC to determine whether the modification introduces additional information about USER_MESSAGE. If introduced, issue a warning.
+* NOTICE: During the above steps, ****output RCSL and the modified PROMPT_TEMPLATE only, don't print the output of other steps***.
+
+----
+
+Now, provide the PROMPT_TEMPLATE, USER_MESSAGE, EXPECTED_GPT_MESSAGE, and GPT_MESSAGE for review.
+
+'''
+
+DEFAULT_CURRENT_SYSTEM_PROMPT = ''
+DEFAULT_OUTPUT_EVALUATING_PROMPT = 'Find out which is more similar to string S, A or B? Print nothing if there\'s no significant difference between A and B. Else, print the result (letter A or B) only. Do nothing else.'
+
+class PromptUI:
+    def __init__(self, advanced_mode = False):
+        self.advanced_mode = advanced_mode
+        self.ui = self.init_ui()
+
+    def init_ui(self):
+        with gr.Blocks() as prompt_ui:            
+            with gr.Row():
+                with gr.Column():
+                    self.testing_user_prompt_textbox = gr.Textbox(label="Testing User Prompt", lines=10, interactive=True)
+                    self.expect_output_textbox = gr.Textbox(label="Expected Output", lines=5, interactive=True)
+                    # Add gr.Number here for iterations input
+                    self.iterations_number = gr.Number(value=1, label="Optimize Iterations", min=1, max=1000, step=1, decimals=0)
+                    # Add button to trigger optimization here
+                    self.optimize_btn = gr.Button(value="Optimize Prompt", variant='primary')
+                    self.similar_candidate_textbox = gr.Textbox(label="Similarity Delta", lines=1, interactive=True)
+                    self.compare_outputs_btn = gr.Button(value="Compare Outputs")
+
+                with gr.Column():
+                    self.new_system_prompt_textbox = gr.Textbox(label="New System Prompt", lines=5, interactive=True)
+                    self.new_output_textbox = gr.Textbox(label="New Output", lines=5, interactive=True)
+                    with gr.Row():
+                        self.run_meta_btn = gr.Button(value="↑ Single Step Optimize")
+                        self.run_new_btn = gr.Button(value="⟳ Run New")
+                    self.new_system_prompt_changed = gr.Checkbox(
+                        label="New System Prompt Changed",
+                        value=False,
+                        interactive=False
+                        )
+
+                with gr.Column():
+                    self.current_system_prompt_textbox = gr.Textbox(label="Current System Prompt", 
+                                                                    value=DEFAULT_CURRENT_SYSTEM_PROMPT,
+                                                                    lines=5, 
+                                                                    interactive=True)
+                    self.current_output_textbox = gr.Textbox(label="Current Output", lines=5, interactive=True)
+                    with gr.Row():
+                        self.accept_new_btn = gr.Button(value="→ Accept New Prompt")
+                        self.run_current_btn = gr.Button(value="⟳ Run Current")
+
+            with gr.Row(visible=self.advanced_mode):
+                with gr.Column():
+                    self.meta_system_prompt_textbox = gr.Textbox(label="Meta System Prompt", 
+                                                                value=DEFAULT_META_SYSTEM_PROMPT,
+                                                                lines=10, 
+                                                                interactive=True
+                                                                )
+                with gr.Column():
+                    self.merged_meta_prompt_textbox = gr.Textbox(label="Merged Meta System Prompt", 
+                                                                lines=10, 
+                                                                interactive=False,
+                                                                show_copy_button=True
+                                                                )
+                    self.merge_prompt_btn = gr.Button(value="Merge Meta System Prompt")
+                    # self.chatgpt_output_textbox = gr.Textbox(label="Paste ChatGPT Output", 
+                    #                                         lines=10,
+                    #                                         interactive=True
+                    #                                         )
+                    # self.parse_chatgpt_output_btn = gr.Button(value="Parse ChatGPT Output")                                      
+
+            with gr.Row(visible=self.advanced_mode):
+                with gr.Column():
+                    self.llm_model_meta_dropdown = gr.Dropdown(
+                        label="Generating LLM Model", 
+                        choices=gpt_models_not_legacy,
+                        value=DEFAULT_MODEL_FOR_GENERATING,
+                        interactive=True,
+                        allow_custom_value=False
+                    )
+
+                    self.llm_model_meta_temperature_slider = gr.Slider(
+                        minimum=0.0, 
+                        maximum=1.0, 
+                        step=0.01, 
+                        value=0.0,
+                        interactive=True,
+                        label="Generating LLM Model Temperature"
+                    )
+
+                    self.llm_model_meta_max_tokens_slider = gr.Slider(
+                        minimum=256, 
+                        maximum=32000, 
+                        step=256, 
+                        value=0,
+                        interactive=True, 
+                        label="Generating LLM Model Token Limit (0 for auto)"
+                    )
+
+                    self.llm_model_meta_request_timeout_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=600, 
+                        step=5, 
+                        value=600,
+                        interactive=True,
+                        label="Generating LLM Model Timeout"
+                    )
+
+                    self.llm_model_meta_max_retries_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=30, 
+                        step=1, 
+                        value=6,
+                        interactive=True,
+                        label="Generating LLM Model Max Retries"
+                    )
+
+                with gr.Column():
+                    self.llm_model_test_dropdown = gr.Dropdown(
+                        label="Testing LLM Model",
+                        choices=gpt_models_not_legacy,
+                        value=DEFAULT_MODEL_FOR_TESTING,
+                        interactive=True,
+                        allow_custom_value=False
+                    )
+
+                    self.llm_model_test_temperature_slider = gr.Slider(
+                        minimum=0.0,
+                        maximum=1.0, 
+                        step=0.01, 
+                        value=0.0,
+                        interactive=True,
+                        label="Testing LLM Model Temperature"
+                    )
+
+                    self.llm_model_test_max_tokens_slider = gr.Slider(
+                        minimum=256, 
+                        maximum=32000, 
+                        step=256, 
+                        value=0,
+                        interactive=True, 
+                        label="Testing LLM Model Token Limit  (0 for auto)"
+                    )
+
+                    self.llm_model_test_request_timeout_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=600, 
+                        step=5, 
+                        value=600, 
+                        interactive=True,
+                        label="Testing LLM Model Timeout"
+                    )
+
+                    self.llm_model_test_max_retries_slider = gr.Slider(
+                        minimum=0, 
+                        maximum=30, 
+                        step=1, 
+                        value=6, 
+                        interactive=True,
+                        label="Testing LLM Model Max Retries"
+                    )
+                # with gr.Column():
+                #     self.llm_model_output_eval_dropdown = gr.Dropdown(label="Output Evaluating LLM Model",
+                #                                                       choices=gpt_models_legacy,
+                #                                                       value=DEFAULT_MODEL_FOR_OUTPUT_EVALUATING,
+                #                                                       interactive=True,
+                #                                                       allow_custom_value=False)
+                #     self.llm_model_output_eval_slider = gr.Slider(minimum=0.0,
+                #                                                          maximum=1.0,
+                #                                                          step=0.01,
+                #                                                          default=0.0, 
+                #                                                          label="Output Evaluating LLM Model of Temperature")
+
+
+            self.run_new_btn.click(
+                self.test_prompt, 
+                [
+                    self.new_system_prompt_textbox,
+                    self.testing_user_prompt_textbox,
+                    self.llm_model_test_dropdown,
+                    self.llm_model_test_max_retries_slider,
+                    self.llm_model_test_max_tokens_slider,
+                    self.llm_model_test_request_timeout_slider,
+                    self.llm_model_test_temperature_slider
+                ], 
+                [self.new_output_textbox]
+            )
+            self.run_current_btn.click(
+                self.test_prompt, 
+                [
+                    self.current_system_prompt_textbox, 
+                    self.testing_user_prompt_textbox,
+                    self.llm_model_test_dropdown,
+                    self.llm_model_test_max_retries_slider,
+                    self.llm_model_test_max_tokens_slider,
+                    self.llm_model_test_request_timeout_slider,
+                    self.llm_model_test_temperature_slider
+                ], 
+                [self.current_output_textbox]
+            )
+            self.run_meta_btn.click(
+                self.meta_prompt, 
+                [
+                    self.meta_system_prompt_textbox, 
+                    self.current_system_prompt_textbox,
+                    self.testing_user_prompt_textbox,
+                    self.expect_output_textbox,
+                    self.current_output_textbox,
+                    self.llm_model_meta_dropdown,
+                    self.llm_model_meta_max_retries_slider,
+                    self.llm_model_meta_max_tokens_slider,
+                    self.llm_model_meta_request_timeout_slider,
+                    self.llm_model_meta_temperature_slider
+                ], 
+                [self.new_system_prompt_textbox, self.new_system_prompt_changed]
+            )
+            self.accept_new_btn.click(self.copy_new_prompts,
+                                      [self.new_system_prompt_textbox, self.new_output_textbox],
+                                      [self.current_system_prompt_textbox, self.current_output_textbox])
+            self.compare_outputs_btn.click(self.compare_outputs,
+                                           [self.new_output_textbox, self.current_output_textbox, self.expect_output_textbox],
+                                           [self.similar_candidate_textbox])
+            # Attach the optimize_prompt function to the button click event.
+            # You should implement this function according to your optimization logic.
+            self.optimize_btn.click(
+                self.optimize_prompt,
+                [
+                    self.meta_system_prompt_textbox, 
+                    self.current_system_prompt_textbox,
+                    self.testing_user_prompt_textbox,
+                    self.expect_output_textbox,
+                    self.current_output_textbox,
+                    self.iterations_number,
+                    self.llm_model_meta_dropdown,
+                    self.llm_model_meta_max_retries_slider,
+                    self.llm_model_meta_max_tokens_slider,
+                    self.llm_model_meta_request_timeout_slider,
+                    self.llm_model_meta_temperature_slider,
+                    self.llm_model_test_dropdown,
+                    self.llm_model_test_max_retries_slider,
+                    self.llm_model_test_max_tokens_slider,
+                    self.llm_model_test_request_timeout_slider,
+                    self.llm_model_test_temperature_slider
+                ],
+                [self.new_system_prompt_textbox, self.new_system_prompt_changed])
+            
+            self.merge_prompt_btn.click(self.merge_meta_system_prompt,
+                                        [
+                                            self.meta_system_prompt_textbox, 
+                                            self.current_system_prompt_textbox,
+                                            self.testing_user_prompt_textbox, 
+                                            self.expect_output_textbox, 
+                                            self.current_output_textbox                                            
+                                        ],
+                                        [self.merged_meta_prompt_textbox])
+
+
+        return prompt_ui
+    
+    def merge_meta_system_prompt(
+            self, 
+            meta_system_prompt, 
+            current_system_prompt,
+            testing_user_prompt, 
+            expect_output, 
+            current_output             
+            ):
+        """Merge meta and current system prompts."""
+
+        user_prompt = self.generate_user_message(
+            current_system_prompt, 
+            testing_user_prompt, 
+            expect_output, 
+            current_output
+        )
+
+        merged_prompt = f"{meta_system_prompt}\n\n{user_prompt}"
+
+        return merged_prompt
+
+    def copy_new_prompts(self, system_prompt, output):
+        """Copy prompts and output from new to current textboxes."""
+
+        return system_prompt, output
+
+    def test_prompt(
+            self, 
+            system_prompt, 
+            user_prompt,
+            model,
+            max_retries,
+            max_tokens,
+            request_timeout,
+            temperature,
+            ):
+        # Create the prompt
+        prompt = [
+            SystemMessage(content=system_prompt),
+            HumanMessage(content=user_prompt)
+        ]
+
+        chat_llm = ChatOpenAI(
+            model=model,
+            max_retries=max_retries,
+            max_tokens=None if max_tokens == 0 else max_tokens,
+            request_timeout=request_timeout,
+            temperature=temperature
+        )
+
+        # Get the response from OpenAI
+        gpt_response = chat_llm(prompt)
+
+        # Return the output to be placed in the output textbox
+        return gpt_response.content
+
+    def generate_user_message(self, current_system_prompt, testing_user_prompt, expect_output, current_output):
+        user_message = f"""
+* Prompt Template
+
+```
+{current_system_prompt}
+```
+
+* User Message
+
+```
+{testing_user_prompt}
+```
+
+* Expected GPT Message
+
+```
+{expect_output}
+```
+
+* GPT Message
+
+```
+{current_output}
+```
+"""
+        return user_message
+
+    def meta_prompt(
+            self,
+            meta_system_prompt, current_system_prompt, testing_user_prompt, expect_output, current_output,
+            model,
+            max_retries,
+            max_tokens,
+            request_timeout,
+            temperature,
+            ):
+        # Format the user message
+#         user_message = f"""
+# * Prompt Template
+
+# ```
+# {current_system_prompt}
+# ```
+
+# * User Message
+
+# ```
+# {testing_user_prompt}
+# ```
+
+# * Expected GPT Message
+
+# ```
+# {expect_output}
+# ```
+
+# * GPT Message
+
+# ```
+# {current_output}
+# ```
+# """
+
+        # Format the user message
+        user_message = self.generate_user_message(
+            current_system_prompt, testing_user_prompt, expect_output, current_output
+        )
+
+        # Create the prompt
+        prompt = [
+            SystemMessage(content=meta_system_prompt),
+            HumanMessage(content=user_message)
+        ]
+
+        chat_llm = ChatOpenAI(
+            model=model,
+            max_retries=max_retries,
+            max_tokens=None if max_tokens == 0 else max_tokens,
+            request_timeout=request_timeout,
+            temperature=temperature
+        )
+
+        # Get the response from OpenAI
+        gpt_response = chat_llm(prompt)
+
+        updated_prompt = self.extract_updated_prompt(gpt_response.content)
+        changed = not self.detect_no_change(gpt_response.content)
+
+        # Return the output to be placed in the new system prompt textbox
+        if updated_prompt:
+            return updated_prompt, changed
+        else:
+            return gpt_response.content, changed
+
+    def extract_updated_prompt(self, gpt_response):
+        # Regular expression pattern to find the text enclosed
+        pattern = "<!-- BEGIN OF PROMPT -->(.*?)<!-- END OF PROMPT -->"
+        
+        # Using search method to find the first occurrence of the pattern
+        result = re.search(pattern, gpt_response, re.DOTALL)
+        
+        if result:
+            s = result.group(1).strip("\n")
+            if s.startswith("```") and s.endswith("```"):
+                s = s[3:-3]        
+            return s # Return the matched string
+        else:
+            return None  # If no such pattern is found return None
+
+    def detect_no_change(self, gpt_response):
+        # Regular expression pattern to find the exact string
+        pattern = "<!-- NO CHANGE TO PROMPT -->"
+        
+        # Using search method to find the occurrence of the pattern
+        result = re.search(pattern, gpt_response)
+        
+        if result:
+            return True  # If the pattern is found return True
+        else:
+            return False  # If no such pattern is found return False
+        
+    # def compare_strings(self, a: str, b: str, s: str) -> str:
+    #     # Create an instance of ChatOpenAI with the evaluation model
+    #     chat_model = OpenAI(temperature=0, model_name=self.llm_model_output_eval_dropdown.value)
+
+    #     # Create a prompt for comparison
+    #     prompt = (DEFAULT_OUTPUT_EVALUATING_PROMPT + 
+    #             '\n\n' + f'# S\n\n```\n{s}\n```\n\n# A\n\n```\n{a}\n```\n\n# B\n\n```\n{b}\n```\n\n')
+
+    #     # Get the response from OpenAI
+    #     response = chat_model(prompt)
+
+    #     # Remove '```' from beginning and end if it exists
+    #     if response.startswith("```") and response.endswith("```"):
+    #         response = response[3:-3]
+
+    #     # Check the first character of the response and return accordingly
+    #     if response.startswith('A'):
+    #         return 'A'
+    #     elif response.startswith('B'):
+    #         return 'B'
+    #     else:
+    #         return None
+
+    def optimize_prompt(
+            self, 
+            meta_system_prompt, 
+            current_system_prompt, 
+            testing_user_prompt, 
+            expect_output, 
+            current_output, 
+            iterations,
+            meta_model,
+            meta_max_retries,
+            meta_max_tokens,
+            meta_request_timeout,
+            meta_temperature,
+            test_model,
+            test_max_retries,
+            test_max_tokens,
+            test_request_timeout,
+            test_temperature,
+            ):
+
+        changed = False
+
+        # Iterate the specified number of times
+        for i in range(int(iterations)):
+            # If current_output is None or not provided, get it from test_prompt
+            if current_output is None:
+                current_output = self.test_prompt(
+                    current_system_prompt, 
+                    testing_user_prompt,
+                    test_model,
+                    test_max_retries,
+                    test_max_tokens,
+                    test_request_timeout,
+                    test_temperature,
+                    )
+
+            # Call meta_prompt to get an optimized prompt
+            new_prompt, changed = self.meta_prompt(
+                meta_system_prompt, 
+                current_system_prompt, 
+                testing_user_prompt, 
+                expect_output, 
+                current_output,
+                meta_model,
+                meta_max_retries,
+                meta_max_tokens,
+                meta_request_timeout,
+                meta_temperature,
+                )
+
+            # If changed is False, break the loop
+            if not changed:
+                break
+
+            # If there is an updated prompt and it's different from the current one, update current_system_prompt
+            if new_prompt and new_prompt != current_system_prompt:
+                current_system_prompt = new_prompt
+                # Reset current_output to None so it gets recalculated in the next iteration
+                current_output = None
+
+        return current_system_prompt, changed  # Return the optimized system prompt
+
+    def compare_strings(self, alpha: str, beta: str, expected: str) -> str:
+        # If both ALPHA and BETA are empty, return None
+        if not alpha and not beta:
+            return None
+
+        # If either ALPHA or BETA is empty, the non-empty string should be considered more similar to EXPECTED
+        if not alpha:
+            return 'B'
+        if not beta:
+            return 'A'
+
+        # If both ALPHA and BETA are identical, return None
+        if alpha == beta:
+            return None
+
+        # Create the CountVectorizer instance
+        vectorizer = CountVectorizer().fit_transform([alpha, beta, expected])
+        vectors = vectorizer.toarray()
+
+        # Calculate cosine similarities
+        alpha_sim = cosine_similarity(vectors[0].reshape(1, -1), vectors[2].reshape(1, -1))
+        beta_sim = cosine_similarity(vectors[1].reshape(1, -1), vectors[2].reshape(1, -1))
+
+        # Compare similarities and return the string that is more similar to the expected string
+        if alpha_sim > beta_sim:
+            return 'A'
+        elif beta_sim > alpha_sim:
+            return 'B'
+        else:
+            return None
+        
+    def delta_similarities(self, alpha: str, beta: str, expected: str) -> float:
+        # If both ALPHA and BETA are empty, return 0
+        if not alpha and not beta:
+            return 0.0
+
+        # If either ALPHA or BETA is empty, the non-empty string should be considered more similar to EXPECTED
+        if not alpha:
+            return -1.0
+        if not beta:
+            return 1.0
+
+        # If both ALPHA and BETA are identical, return 0
+        if alpha == beta:
+            return 0.0
+
+        # Create the CountVectorizer instance
+        vectorizer = CountVectorizer().fit_transform([alpha, beta, expected])
+        vectors = vectorizer.toarray()
+
+        # Calculate cosine similarities
+        alpha_sim = cosine_similarity(vectors[0].reshape(1, -1), vectors[2].reshape(1, -1))
+        beta_sim = cosine_similarity(vectors[1].reshape(1, -1), vectors[2].reshape(1, -1))
+
+        # Return the difference in similarities
+        return alpha_sim[0][0] - beta_sim[0][0]
+
+    def compare_outputs(self, new_output, current_output, expected_output):
+        # Compare new output and current output against expected output
+        # result = self.compare_strings(new_output, current_output, expected_output)
+        result = self.delta_similarities(new_output, current_output, expected_output)
+
+        return result

requirements.txt

@@ -0,0 +1,80 @@
+aiofiles==23.2.1
+aiohttp==3.8.5
+aiosignal==1.3.1
+altair==5.1.1
+annotated-types==0.5.0
+anyio==3.7.1
+async-timeout==4.0.3
+attrs==23.1.0
+certifi==2023.7.22
+charset-normalizer==3.2.0
+click==8.1.7
+contourpy==1.1.1
+cycler==0.11.0
+dataclasses-json==0.6.0
+exceptiongroup==1.1.3
+fastapi==0.103.1
+ffmpy==0.3.1
+filelock==3.12.4
+fonttools==4.42.1
+frozenlist==1.4.0
+fsspec==2023.9.2
+gradio==3.44.4
+gradio_client==0.5.1
+greenlet==2.0.2
+h11==0.14.0
+httpcore==0.18.0
+httpx==0.25.0
+huggingface-hub==0.17.2
+idna==3.4
+importlib-resources==6.1.0
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+langchain==0.0.300
+langsmith==0.0.40
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+numexpr==2.8.6
+numpy==1.26.0
+openai==0.28.0
+orjson==3.9.7
+packaging==23.1
+pandas==2.1.1
+Pillow==10.0.1
+pydantic==2.3.0
+pydantic_core==2.6.3
+pydub==0.25.1
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.30.2
+requests==2.31.0
+rpds-py==0.10.3
+scikit-learn==1.3.1
+scipy==1.11.3
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.21
+starlette==0.27.0
+tenacity==8.2.3
+threadpoolctl==3.2.0
+toolz==0.12.0
+tqdm==4.66.1
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==2.0.5
+uvicorn==0.23.2
+websockets==11.0.3
+yarl==1.9.2