Added input dimension analysis.

demo/sample_generator.ipynb

@@ -2,67 +2,86 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
-    "\n",
-    "# Load configuration from YAML file\n",
-    "config = {\n",
-    "    \"model_name\": \"llama3-70b-8192\",\n",
-    "    # \"model_name\": \"llama3-8b-8192\",\n",
-    "    # \"model_name\": \"llama-3.1-70b-versatile\",\n",
-    "    # \"model_name\": \"llama-3.1-8b-instant\",\n",
-    "    # \"model_name\": \"gemma2-9b-it\",\n",
-    "}\n"
+    "# import os\n",
+    "\n",
+    "# # Load configuration from YAML file\n",
+    "# config = {\n",
+    "#     \"model_name\": \"llama3-70b-8192\",\n",
+    "#     # \"model_name\": \"llama3-8b-8192\",\n",
+    "#     # \"model_name\": \"llama-3.1-70b-versatile\",\n",
+    "#     # \"model_name\": \"llama-3.1-8b-instant\",\n",
+    "#     # \"model_name\": \"gemma2-9b-it\",\n",
+    "# }\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import json\n",
-    "import yaml\n",
-    "from langchain.prompts import ChatPromptTemplate\n",
-    "from langchain.chat_models import ChatOpenAI\n",
-    "from langchain.schema.output_parser import StrOutputParser\n",
-    "from langchain_core.runnables import RunnablePassthrough, RunnableParallel, RunnableLambda\n",
-    "from langchain_core.output_parsers import JsonOutputParser\n",
-    "from langchain.output_parsers import YamlOutputParser\n",
-    "\n",
     "# Define prompt strings as constants\n",
     "DESCRIPTION_PROMPT = [\n",
-    "    (\"system\", \"\"\"Given the following JSON example for a task type:\n",
+    "    (\"system\", \"\"\"Given the following JSON example(s) for a task type:\n",
+    "     \n",
     "{raw_example}\n",
     "\n",
-    "Provide a concise description of the task type, including the format and style of the output.\n",
+    "Provide a concise description of the task type, including the format and\n",
+    "style of the output. If there are multiple examples, provide a description\n",
+    "for the task type as a whole, ignore the unique parts of the examples.\n",
     "\n",
     "Format your response as follows:\n",
     "Task Description: [Your description here]\n",
     "\"\"\")\n",
     "]\n",
     "\n",
+    "INPUT_ANALYSIS_PROMPT = [\n",
+    "    (\"system\", \"\"\"Describe input dimensions, attributes, ranges, and typical values\n",
+    "for a specific task type. Identify main inputs, their impacts, and interactions.\n",
+    "Provide names, descriptions, ranges, and examples for each. Explain how they\n",
+    "affect task execution or results. Include an example of generating comprehensive\n",
+    "input samples using these dimensions and attributes.\n",
+    "\n",
+    "Format your response as follows:\n",
+    "Input Analysis: [Your analysis here]\n",
+    "\"\"\"),\n",
+    "    (\"user\", \"\"\"Task Description:\n",
+    "\n",
+    "{description}\n",
+    "\n",
+    "\"\"\")\n",
+    "]\n",
+    "\n",
     "BRIEFS_PROMPT = [\n",
-    "    (\"system\", \"\"\"Given the task type description, generate descriptions for {generating_batch_size} new \n",
-    "examples with detailed attributes based on this task type. But don't provide any detailed task output.\n",
+    "    (\"system\", \"\"\"Given the task type description, and input analysis, generate\n",
+    "descriptions for {generating_batch_size} new examples with detailed attributes\n",
+    "based on this task type. But don't provide any detailed task output.\n",
+    "\n",
+    "Use the input analysis to create diverse and comprehensive example briefs that\n",
+    "cover various input dimensions and attribute ranges.\n",
     "\n",
-    "Format your response as a valid YAML object with a single key\n",
-    "'new_example_briefs' containing a YAML array of {generating_batch_size} objects, each\n",
-    "with a 'example_brief' field.\n",
+    "Format your response as a valid YAML object with a single key 'new_example_briefs'\n",
+    "containing a YAML array of {generating_batch_size} objects, each with a\n",
+    "'example_brief' field.\n",
     "\"\"\"),\n",
     "    (\"user\", \"\"\"Task Description:\n",
     "\n",
     "{description}\n",
     "\n",
+    "Input Analysis:\n",
+    "\n",
+    "{input_analysis}\n",
+    "\n",
     "\"\"\")\n",
     "]\n",
     "\n",
     "EXAMPLES_FROM_BRIEFS_PROMPT = [\n",
     "    (\"system\", \"\"\"Given the task type description, brief descriptions for new examples, \n",
-    "and JSON example, generate 3 more input/output examples for this task type,\n",
+    "and JSON example(s), generate {generating_batch_size} more input/output examples for this task type,\n",
     "strictly based on the brief descriptions. Ensure that the new examples are\n",
     "consistent with the brief descriptions and do not introduce any new information\n",
     "not present in the briefs.\n",
@@ -78,7 +97,7 @@
     "\n",
     "{new_example_briefs}\n",
     "\n",
-    "Example:\n",
+    "Example(s):\n",
     "\n",
     "{raw_example}\n",
     "\n",
@@ -86,7 +105,7 @@
     "]\n",
     "\n",
     "EXAMPLES_PROMPT = [\n",
-    "    (\"system\", \"\"\"Given the task type description, and input/output example, generate {generating_batch_size}\n",
+    "    (\"system\", \"\"\"Given the task type description, and input/output example(s), generate {generating_batch_size}\n",
     "new input/output examples for this task type.\n",
     "\n",
     "Format your response as a valid JSON object with a single key 'examples' \n",
@@ -96,16 +115,34 @@
     "\n",
     "{description}\n",
     "\n",
-    "Example:\n",
+    "Example(s):\n",
     "\n",
     "{raw_example}\n",
     "\n",
     "\"\"\")\n",
-    "]\n",
+    "]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import yaml\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from langchain_core.runnables import RunnablePassthrough, RunnableParallel, RunnableLambda\n",
+    "from langchain_core.output_parsers import JsonOutputParser\n",
+    "from langchain.output_parsers import YamlOutputParser\n",
+    "\n",
     "\n",
     "class TaskDescriptionGenerator:\n",
     "    def __init__(self, model):        \n",
     "        self.description_prompt = ChatPromptTemplate.from_messages(DESCRIPTION_PROMPT)\n",
+    "        self.input_analysis_prompt = ChatPromptTemplate.from_messages(INPUT_ANALYSIS_PROMPT)\n",
     "        self.briefs_prompt = ChatPromptTemplate.from_messages(BRIEFS_PROMPT)\n",
     "        self.examples_from_briefs_prompt = ChatPromptTemplate.from_messages(EXAMPLES_FROM_BRIEFS_PROMPT)\n",
     "        self.examples_prompt = ChatPromptTemplate.from_messages(EXAMPLES_PROMPT)\n",
@@ -116,6 +153,7 @@
     "        json_parse = JsonOutputParser()\n",
     "\n",
     "        self.description_chain = self.description_prompt | model | output_parser\n",
+    "        self.input_analysis_chain = self.input_analysis_prompt | model | output_parser\n",
     "        self.briefs_chain = self.briefs_prompt | model | output_parser\n",
     "        self.examples_from_briefs_chain = self.examples_from_briefs_prompt | json_model | json_parse\n",
     "        self.examples_chain = self.examples_prompt | json_model | json_parse\n",
@@ -125,7 +163,9 @@
     "            | RunnablePassthrough.assign(description = self.description_chain)\n",
     "            | {\n",
     "                \"description\": lambda x: x[\"description\"],\n",
-    "                \"examples_from_briefs\": RunnablePassthrough.assign(new_example_briefs = lambda x: self.briefs_chain.invoke(x)) | self.examples_from_briefs_chain,\n",
+    "                \"examples_from_briefs\": RunnablePassthrough.assign(input_analysis = lambda x: self.input_analysis_chain.invoke(x))\n",
+    "                    | RunnablePassthrough.assign(new_example_briefs = lambda x: self.briefs_chain.invoke(x)) \n",
+    "                    | self.examples_from_briefs_chain,\n",
     "                \"examples\": self.examples_chain\n",
     "            }\n",
     "            | RunnablePassthrough.assign(\n",
@@ -138,9 +178,20 @@
     "\n",
     "    def process(self, input_str, generating_batch_size=3):\n",
     "        try:\n",
-    "            # Parse input string to a dictionary\n",
-    "            example_dict = json.loads(input_str) if input_str.startswith('{') else yaml.safe_load(input_str)\n",
-    "            if not isinstance(example_dict, dict) or 'input' not in example_dict or 'output' not in example_dict:\n",
+    "            try:\n",
+    "                example_dict = json.loads(input_str)\n",
+    "            except ValueError:\n",
+    "                try:\n",
+    "                    example_dict = yaml.safe_load(input_str)\n",
+    "                except yaml.YAMLError as e:\n",
+    "                    raise ValueError(\"Invalid input format. Expected a JSON or YAML object.\") from e\n",
+    "\n",
+    "            # If example_dict is a list, filter out invalid items\n",
+    "            if isinstance(example_dict, list):\n",
+    "                example_dict = [item for item in example_dict if isinstance(item, dict) and 'input' in item and 'output' in item]\n",
+    "\n",
+    "            # If example_dict is not a list, check if it's a valid dict\n",
+    "            elif not isinstance(example_dict, dict) or 'input' not in example_dict or 'output' not in example_dict:\n",
     "                raise ValueError(\"Invalid input format. Expected an object with 'input' and 'output' fields.\")\n",
     "\n",
     "            # Move the original content to a key named 'example'\n",
@@ -156,9 +207,47 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/yale/work/meta-prompt/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7861\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7861/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/yale/work/meta-prompt/.venv/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:141: LangChainDeprecationWarning: The class `ChatOpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n",
+      "  warn_deprecated(\n"
+     ]
+    }
+   ],
    "source": [
     "import gradio as gr\n",
     "\n",