Delete app/cookies_llm_experimental_eval.ipynb

app/cookies_llm_experimental_eval.ipynb

@@ -1,412 +0,0 @@
-{
- "cells": [
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## This notebook is used to evaluate the performance of the LLM model to clean the cookies dataset."
-   ]
-  },
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### Using OpenAI API"
-   ]
-  },
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Try to call function from newly created module `openai_chat_completion.py`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{\n",
-      "  \"id\": \"chatcmpl-7UecmbvjkbwamEpYXyIvGO4w4Ae8i\",\n",
-      "  \"object\": \"chat.completion\",\n",
-      "  \"created\": 1687540356,\n",
-      "  \"model\": \"gpt-4-0314\",\n",
-      "  \"choices\": [\n",
-      "    {\n",
-      "      \"index\": 0,\n",
-      "      \"message\": {\n",
-      "        \"role\": \"assistant\",\n",
-      "        \"content\": \"Chill Medicated,Edible,Beverage,Watermelon,250\"\n",
-      "      },\n",
-      "      \"finish_reason\": \"stop\"\n",
-      "    }\n",
-      "  ],\n",
-      "  \"usage\": {\n",
-      "    \"prompt_tokens\": 636,\n",
-      "    \"completion_tokens\": 15,\n",
-      "    \"total_tokens\": 651\n",
-      "  }\n",
-      "}\n"
-     ]
-    }
-   ],
-   "source": [
-    "import os\n",
-    "from dotenv import load_dotenv\n",
-    "load_dotenv()\n",
-    "\n",
-    "import openai\n",
-    "\n",
-    "# set OPENAI_API_KEY environment variable from .env file\n",
-    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
-    "\n",
-    "# import OpenAIChatCompletions class from openai_chat_completion.py file located in llm_data_cleaner/scripts folder (this notebook is located in llm_data_cleaner/notebooks folder)\n",
-    "from openai_chat_completion import OpenAIChatCompletions, compare_completion_and_prediction\n",
-    "\n",
-    "# read in llm-data-cleaner/prompts/gpt4-system-message.txt file into variable system_message\n",
-    "system_message = open('../prompts/gpt4-system-message.txt', 'r').read()\n",
-    "\n",
-    "# create an instance of the OpenAIChatCompletions class and use the openai_chat_completion method to get chat completions\n",
-    "chat_completions = OpenAIChatCompletions(model=\"gpt-4\", system_message=system_message)\n",
-    "\n",
-    "prompt = \"co-2MFE5QVF,Chill Medicated - Watermelon - Syrup - 250mg,Chill Medicated,nan,nan,nan\"\n",
-    "completion = chat_completions.openai_chat_completion(prompt, n_shot=1)\n",
-    "print(completion)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from openai_chat_completion import OpenAIChatCompletions\n",
-    "\n",
-    "test_prompts, test_completions, test_predictions = chat_completions.predict_jsonl(n_shot=1)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>prompt</th>\n",
-       "      <th>completion</th>\n",
-       "      <th>prediction</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>co-2MFE5QVF,Chill Medicated - Watermelon - Syr...</td>\n",
-       "      <td>Chill Medicated,Edible,Beverage,nan,nan</td>\n",
-       "      <td>Chill Medicated,Edible,Beverage,Watermelon,250</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>bl-111630024545,Feelz - Space Cowboy 3.5g,nan,...</td>\n",
-       "      <td>Feelz,Flower,Bud,Space Cowboy,3.5</td>\n",
-       "      <td>Feelz,Flower,Bud,Space Cowboy,3.5</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>fl-8voAjt83sD,Champelli | Xclusivo 3.5g | Eigh...</td>\n",
-       "      <td>Champelli,Flower,Bud,Xclusivo,3.5</td>\n",
-       "      <td>Champelli,Flower,Bud,Xclusivo,3.5</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>bl-073133213364,CAM - Mellowz #7 7g,nan,FLOWER...</td>\n",
-       "      <td>CAM,Flower,Bud,Mellowz #7,7</td>\n",
-       "      <td>CAM - Mellowz #7 7g,CAM,Flower,Bud,Mellowz,7</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>fl-fwJQL2AWnS,Backpack Boyz | Bubblegum Gelato...</td>\n",
-       "      <td>Backpack Boyz,Edible,CBD Tincture/Caps/etc,nan...</td>\n",
-       "      <td>Backpack Boyz,Edible,Syrup,Bubblegum Gelato,1</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5</th>\n",
-       "      <td>fl-dXE5cH45AQ,Raw Garden | Pink Lemonade Crush...</td>\n",
-       "      <td>Raw Garden,Concentrate,Diamonds,Pink Lemonade,1</td>\n",
-       "      <td>Raw Garden,Concentrate,Diamonds,Pink Lemonade,1.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>6</th>\n",
-       "      <td>md-1159983,Baby Jeeter Peaches | Infused Prero...</td>\n",
-       "      <td>Jeeter,Preroll,Infused Joint,Peaches,12.5</td>\n",
-       "      <td>Jeeter,Preroll,Infused Joint,Peaches,12.5</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>7</th>\n",
-       "      <td>co-6WGV1Z0H,Ocean Breeze Cultivators - Truffle...</td>\n",
-       "      <td>Ocean Breeze Cultivators,Preroll,Blunt,Truffle...</td>\n",
-       "      <td>Ocean Breeze Cultivators,Blunt,Truffle Pupil,1.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>8</th>\n",
-       "      <td>fl-doWkMzvFq2,Cookies | Triple Scoop Preroll 1...</td>\n",
-       "      <td>Cookies,Preroll,Joint,Triple Scoop,1</td>\n",
-       "      <td>Cookies,Preroll,Joint,Triple Scoop,1.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9</th>\n",
-       "      <td>bl-842922110296,STIIIZY - Birthday Cake Pod 1g...</td>\n",
-       "      <td>STIIIZY,Vape,Vape,Birthday Cake,1</td>\n",
-       "      <td>Birthday Cake Pod,STIIIZY,Vape,Birthday Cake,1.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>10</th>\n",
-       "      <td>co-6GGKA0GV,Cookies x The Grower Circle - Tang...</td>\n",
-       "      <td>Cookies,Preroll,Infused Joint,TangEray,1</td>\n",
-       "      <td>Here is the missing information:\\n\\nproduct_na...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>11</th>\n",
-       "      <td>co-7YGGNHBR,Kanha - Nano Blood Orange Bliss In...</td>\n",
-       "      <td>Kanha,Edible,Gummies,nan,nan</td>\n",
-       "      <td>Kanha,Edible,Gummies,nan,nan</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>12</th>\n",
-       "      <td>md-1242517,BOBBI HYLL | Indoor - 3.5g,Fresca,F...</td>\n",
-       "      <td>Fresca,Flower,Bud,BOBBI HYLL,3.5</td>\n",
-       "      <td>Fresca,Flower,Bud,BOBBI HYLL,3.5</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>13</th>\n",
-       "      <td>fl-9nEPGsnYtY,Flav | Blueberry Belts 100mg,Fla...</td>\n",
-       "      <td>Flav,Edible,Gummies,nan,nan</td>\n",
-       "      <td>Flav,Edible,Gummies,Blueberry,100</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>14</th>\n",
-       "      <td>md-1347497,Cherry Pie - 1g - Pod,Left Coast Ex...</td>\n",
-       "      <td>Left Coast Extracts,Vape,510 cart,Cherry Pie,1</td>\n",
-       "      <td>Cherry Pie - 1g - Pod,Left Coast Extracts,Vape...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>15</th>\n",
-       "      <td>co-76GP441T,Minntz - Emerald Cut - Indoor - Jo...</td>\n",
-       "      <td>Minntz,Preroll,Joint,Emerald Cut,1</td>\n",
-       "      <td>Minntz,Preroll,Joint,Emerald Cut,1</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16</th>\n",
-       "      <td>co-5RAWYHYQ,The Growers Circle - Double Down -...</td>\n",
-       "      <td>The Growers Circle,Flower,Bud,Double Down,3.5</td>\n",
-       "      <td>The Growers Circle,Flower,Bud,Double Down,3.5</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>17</th>\n",
-       "      <td>md-1195389,Blue Dream Roll Your Own Sugar Shak...</td>\n",
-       "      <td>Pacific Stone,Flower,Bud,nan,14</td>\n",
-       "      <td>Pacific Stone,Flower,Bud,Blue Dream,14</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>18</th>\n",
-       "      <td>co-847ZXF37,The Grower Circle - Zoo Dawg x Cos...</td>\n",
-       "      <td>The Growers Circle,Preroll,Joint,Zoo Dawg x Co...</td>\n",
-       "      <td>The Grower Circle,Preroll,Joint,Zoo Dawg x Cos...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>19</th>\n",
-       "      <td>co-8EMW15ZM,Flight Bites - S'mores - Gummy - 1...</td>\n",
-       "      <td>Flight Bites,Edible,Gummies,nan,nan</td>\n",
-       "      <td>Flight Bites,Edible,Gummies,S'mores,10</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                               prompt  \\\n",
-       "0   co-2MFE5QVF,Chill Medicated - Watermelon - Syr...   \n",
-       "1   bl-111630024545,Feelz - Space Cowboy 3.5g,nan,...   \n",
-       "2   fl-8voAjt83sD,Champelli | Xclusivo 3.5g | Eigh...   \n",
-       "3   bl-073133213364,CAM - Mellowz #7 7g,nan,FLOWER...   \n",
-       "4   fl-fwJQL2AWnS,Backpack Boyz | Bubblegum Gelato...   \n",
-       "5   fl-dXE5cH45AQ,Raw Garden | Pink Lemonade Crush...   \n",
-       "6   md-1159983,Baby Jeeter Peaches | Infused Prero...   \n",
-       "7   co-6WGV1Z0H,Ocean Breeze Cultivators - Truffle...   \n",
-       "8   fl-doWkMzvFq2,Cookies | Triple Scoop Preroll 1...   \n",
-       "9   bl-842922110296,STIIIZY - Birthday Cake Pod 1g...   \n",
-       "10  co-6GGKA0GV,Cookies x The Grower Circle - Tang...   \n",
-       "11  co-7YGGNHBR,Kanha - Nano Blood Orange Bliss In...   \n",
-       "12  md-1242517,BOBBI HYLL | Indoor - 3.5g,Fresca,F...   \n",
-       "13  fl-9nEPGsnYtY,Flav | Blueberry Belts 100mg,Fla...   \n",
-       "14  md-1347497,Cherry Pie - 1g - Pod,Left Coast Ex...   \n",
-       "15  co-76GP441T,Minntz - Emerald Cut - Indoor - Jo...   \n",
-       "16  co-5RAWYHYQ,The Growers Circle - Double Down -...   \n",
-       "17  md-1195389,Blue Dream Roll Your Own Sugar Shak...   \n",
-       "18  co-847ZXF37,The Grower Circle - Zoo Dawg x Cos...   \n",
-       "19  co-8EMW15ZM,Flight Bites - S'mores - Gummy - 1...   \n",
-       "\n",
-       "                                           completion  \\\n",
-       "0             Chill Medicated,Edible,Beverage,nan,nan   \n",
-       "1                   Feelz,Flower,Bud,Space Cowboy,3.5   \n",
-       "2                   Champelli,Flower,Bud,Xclusivo,3.5   \n",
-       "3                         CAM,Flower,Bud,Mellowz #7,7   \n",
-       "4   Backpack Boyz,Edible,CBD Tincture/Caps/etc,nan...   \n",
-       "5     Raw Garden,Concentrate,Diamonds,Pink Lemonade,1   \n",
-       "6           Jeeter,Preroll,Infused Joint,Peaches,12.5   \n",
-       "7   Ocean Breeze Cultivators,Preroll,Blunt,Truffle...   \n",
-       "8                Cookies,Preroll,Joint,Triple Scoop,1   \n",
-       "9                   STIIIZY,Vape,Vape,Birthday Cake,1   \n",
-       "10           Cookies,Preroll,Infused Joint,TangEray,1   \n",
-       "11                       Kanha,Edible,Gummies,nan,nan   \n",
-       "12                   Fresca,Flower,Bud,BOBBI HYLL,3.5   \n",
-       "13                        Flav,Edible,Gummies,nan,nan   \n",
-       "14     Left Coast Extracts,Vape,510 cart,Cherry Pie,1   \n",
-       "15                 Minntz,Preroll,Joint,Emerald Cut,1   \n",
-       "16      The Growers Circle,Flower,Bud,Double Down,3.5   \n",
-       "17                    Pacific Stone,Flower,Bud,nan,14   \n",
-       "18  The Growers Circle,Preroll,Joint,Zoo Dawg x Co...   \n",
-       "19                Flight Bites,Edible,Gummies,nan,nan   \n",
-       "\n",
-       "                                           prediction  \n",
-       "0      Chill Medicated,Edible,Beverage,Watermelon,250  \n",
-       "1                   Feelz,Flower,Bud,Space Cowboy,3.5  \n",
-       "2                   Champelli,Flower,Bud,Xclusivo,3.5  \n",
-       "3        CAM - Mellowz #7 7g,CAM,Flower,Bud,Mellowz,7  \n",
-       "4       Backpack Boyz,Edible,Syrup,Bubblegum Gelato,1  \n",
-       "5   Raw Garden,Concentrate,Diamonds,Pink Lemonade,1.0  \n",
-       "6           Jeeter,Preroll,Infused Joint,Peaches,12.5  \n",
-       "7    Ocean Breeze Cultivators,Blunt,Truffle Pupil,1.0  \n",
-       "8              Cookies,Preroll,Joint,Triple Scoop,1.0  \n",
-       "9    Birthday Cake Pod,STIIIZY,Vape,Birthday Cake,1.0  \n",
-       "10  Here is the missing information:\\n\\nproduct_na...  \n",
-       "11                       Kanha,Edible,Gummies,nan,nan  \n",
-       "12                   Fresca,Flower,Bud,BOBBI HYLL,3.5  \n",
-       "13                  Flav,Edible,Gummies,Blueberry,100  \n",
-       "14  Cherry Pie - 1g - Pod,Left Coast Extracts,Vape...  \n",
-       "15                 Minntz,Preroll,Joint,Emerald Cut,1  \n",
-       "16      The Growers Circle,Flower,Bud,Double Down,3.5  \n",
-       "17             Pacific Stone,Flower,Bud,Blue Dream,14  \n",
-       "18  The Grower Circle,Preroll,Joint,Zoo Dawg x Cos...  \n",
-       "19             Flight Bites,Edible,Gummies,S'mores,10  "
-      ]
-     },
-     "execution_count": 44,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import pandas as pd\n",
-    "\n",
-    "test_results = pd.DataFrame(\n",
-    "    zip(test_prompts,\n",
-    "        test_completions,\n",
-    "        [ele['choices'][0]['message']['content'] for ele in test_predictions])\n",
-    "    , columns = ['prompt','completion','prediction'])\n",
-    "test_results"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'completion': ['Chill Medicated', 'Edible', 'Beverage', 'nan', 'nan'],\n",
-       " 'prediction': ['Chill Medicated', 'Edible', 'Beverage', 'Watermelon', '250'],\n",
-       " 'matches': [True, True, True, False, False],\n",
-       " 'num_correct': 3}"
-      ]
-     },
-     "execution_count": 27,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from util import compare_completion_and_prediction\n",
-    "\n",
-    "compare_completion_and_prediction(test_completions[0], test_predictions[0][\"choices\"][0][\"message\"][\"content\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 58,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "66"
-      ]
-     },
-     "execution_count": 58,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# Write function that uses compare_completion_and_prediction to return num_correct and return zero if there is an error\n",
-    "def get_num_correct(completion, prediction):\n",
-    "    try:\n",
-    "        return compare_completion_and_prediction(completion, prediction)['num_correct']\n",
-    "    except:\n",
-    "        return 0 # this will be the case when format is incorrect\n",
-    "\n",
-    "# Apply get_num_correct function to test_results dataframe\n",
-    "test_results['num_correct'] = test_results.apply(lambda row: get_num_correct(row['completion'], row['prediction']), axis=1)\n",
-    "test_results['num_correct'].sum() # out of 100 possible correct predictions (20 samples * 5 cols per sample)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "kd-llm-dc",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.11"
-  },
-  "orig_nbformat": 4
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}