{ "cells": [ { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## This notebook is used to evaluate the performance of the LLM model to clean the cookies dataset." ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "#### Using OpenAI API" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Try to call function from newly created module `openai_chat_completion.py`" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"id\": \"chatcmpl-7UecmbvjkbwamEpYXyIvGO4w4Ae8i\",\n", " \"object\": \"chat.completion\",\n", " \"created\": 1687540356,\n", " \"model\": \"gpt-4-0314\",\n", " \"choices\": [\n", " {\n", " \"index\": 0,\n", " \"message\": {\n", " \"role\": \"assistant\",\n", " \"content\": \"Chill Medicated,Edible,Beverage,Watermelon,250\"\n", " },\n", " \"finish_reason\": \"stop\"\n", " }\n", " ],\n", " \"usage\": {\n", " \"prompt_tokens\": 636,\n", " \"completion_tokens\": 15,\n", " \"total_tokens\": 651\n", " }\n", "}\n" ] } ], "source": [ "import os\n", "from dotenv import load_dotenv\n", "load_dotenv()\n", "\n", "import openai\n", "\n", "# set OPENAI_API_KEY environment variable from .env file\n", "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n", "\n", "# import OpenAIChatCompletions class from openai_chat_completion.py file located in llm_data_cleaner/scripts folder (this notebook is located in llm_data_cleaner/notebooks folder)\n", "from openai_chat_completion import OpenAIChatCompletions, compare_completion_and_prediction\n", "\n", "# read in llm-data-cleaner/prompts/gpt4-system-message.txt file into variable system_message\n", "system_message = open('../prompts/gpt4-system-message.txt', 'r').read()\n", "\n", "# create an instance of the OpenAIChatCompletions class and use the openai_chat_completion method to get chat completions\n", "chat_completions = OpenAIChatCompletions(model=\"gpt-4\", system_message=system_message)\n", "\n", "prompt = \"co-2MFE5QVF,Chill Medicated - Watermelon - Syrup - 250mg,Chill Medicated,nan,nan,nan\"\n", "completion = chat_completions.openai_chat_completion(prompt, n_shot=1)\n", "print(completion)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "from openai_chat_completion import OpenAIChatCompletions\n", "\n", "test_prompts, test_completions, test_predictions = chat_completions.predict_jsonl(n_shot=1)" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
promptcompletionprediction
0co-2MFE5QVF,Chill Medicated - Watermelon - Syr...Chill Medicated,Edible,Beverage,nan,nanChill Medicated,Edible,Beverage,Watermelon,250
1bl-111630024545,Feelz - Space Cowboy 3.5g,nan,...Feelz,Flower,Bud,Space Cowboy,3.5Feelz,Flower,Bud,Space Cowboy,3.5
2fl-8voAjt83sD,Champelli | Xclusivo 3.5g | Eigh...Champelli,Flower,Bud,Xclusivo,3.5Champelli,Flower,Bud,Xclusivo,3.5
3bl-073133213364,CAM - Mellowz #7 7g,nan,FLOWER...CAM,Flower,Bud,Mellowz #7,7CAM - Mellowz #7 7g,CAM,Flower,Bud,Mellowz,7
4fl-fwJQL2AWnS,Backpack Boyz | Bubblegum Gelato...Backpack Boyz,Edible,CBD Tincture/Caps/etc,nan...Backpack Boyz,Edible,Syrup,Bubblegum Gelato,1
5fl-dXE5cH45AQ,Raw Garden | Pink Lemonade Crush...Raw Garden,Concentrate,Diamonds,Pink Lemonade,1Raw Garden,Concentrate,Diamonds,Pink Lemonade,1.0
6md-1159983,Baby Jeeter Peaches | Infused Prero...Jeeter,Preroll,Infused Joint,Peaches,12.5Jeeter,Preroll,Infused Joint,Peaches,12.5
7co-6WGV1Z0H,Ocean Breeze Cultivators - Truffle...Ocean Breeze Cultivators,Preroll,Blunt,Truffle...Ocean Breeze Cultivators,Blunt,Truffle Pupil,1.0
8fl-doWkMzvFq2,Cookies | Triple Scoop Preroll 1...Cookies,Preroll,Joint,Triple Scoop,1Cookies,Preroll,Joint,Triple Scoop,1.0
9bl-842922110296,STIIIZY - Birthday Cake Pod 1g...STIIIZY,Vape,Vape,Birthday Cake,1Birthday Cake Pod,STIIIZY,Vape,Birthday Cake,1.0
10co-6GGKA0GV,Cookies x The Grower Circle - Tang...Cookies,Preroll,Infused Joint,TangEray,1Here is the missing information:\\n\\nproduct_na...
11co-7YGGNHBR,Kanha - Nano Blood Orange Bliss In...Kanha,Edible,Gummies,nan,nanKanha,Edible,Gummies,nan,nan
12md-1242517,BOBBI HYLL | Indoor - 3.5g,Fresca,F...Fresca,Flower,Bud,BOBBI HYLL,3.5Fresca,Flower,Bud,BOBBI HYLL,3.5
13fl-9nEPGsnYtY,Flav | Blueberry Belts 100mg,Fla...Flav,Edible,Gummies,nan,nanFlav,Edible,Gummies,Blueberry,100
14md-1347497,Cherry Pie - 1g - Pod,Left Coast Ex...Left Coast Extracts,Vape,510 cart,Cherry Pie,1Cherry Pie - 1g - Pod,Left Coast Extracts,Vape...
15co-76GP441T,Minntz - Emerald Cut - Indoor - Jo...Minntz,Preroll,Joint,Emerald Cut,1Minntz,Preroll,Joint,Emerald Cut,1
16co-5RAWYHYQ,The Growers Circle - Double Down -...The Growers Circle,Flower,Bud,Double Down,3.5The Growers Circle,Flower,Bud,Double Down,3.5
17md-1195389,Blue Dream Roll Your Own Sugar Shak...Pacific Stone,Flower,Bud,nan,14Pacific Stone,Flower,Bud,Blue Dream,14
18co-847ZXF37,The Grower Circle - Zoo Dawg x Cos...The Growers Circle,Preroll,Joint,Zoo Dawg x Co...The Grower Circle,Preroll,Joint,Zoo Dawg x Cos...
19co-8EMW15ZM,Flight Bites - S'mores - Gummy - 1...Flight Bites,Edible,Gummies,nan,nanFlight Bites,Edible,Gummies,S'mores,10
\n", "
" ], "text/plain": [ " prompt \\\n", "0 co-2MFE5QVF,Chill Medicated - Watermelon - Syr... \n", "1 bl-111630024545,Feelz - Space Cowboy 3.5g,nan,... \n", "2 fl-8voAjt83sD,Champelli | Xclusivo 3.5g | Eigh... \n", "3 bl-073133213364,CAM - Mellowz #7 7g,nan,FLOWER... \n", "4 fl-fwJQL2AWnS,Backpack Boyz | Bubblegum Gelato... \n", "5 fl-dXE5cH45AQ,Raw Garden | Pink Lemonade Crush... \n", "6 md-1159983,Baby Jeeter Peaches | Infused Prero... \n", "7 co-6WGV1Z0H,Ocean Breeze Cultivators - Truffle... \n", "8 fl-doWkMzvFq2,Cookies | Triple Scoop Preroll 1... \n", "9 bl-842922110296,STIIIZY - Birthday Cake Pod 1g... \n", "10 co-6GGKA0GV,Cookies x The Grower Circle - Tang... \n", "11 co-7YGGNHBR,Kanha - Nano Blood Orange Bliss In... \n", "12 md-1242517,BOBBI HYLL | Indoor - 3.5g,Fresca,F... \n", "13 fl-9nEPGsnYtY,Flav | Blueberry Belts 100mg,Fla... \n", "14 md-1347497,Cherry Pie - 1g - Pod,Left Coast Ex... \n", "15 co-76GP441T,Minntz - Emerald Cut - Indoor - Jo... \n", "16 co-5RAWYHYQ,The Growers Circle - Double Down -... \n", "17 md-1195389,Blue Dream Roll Your Own Sugar Shak... \n", "18 co-847ZXF37,The Grower Circle - Zoo Dawg x Cos... \n", "19 co-8EMW15ZM,Flight Bites - S'mores - Gummy - 1... \n", "\n", " completion \\\n", "0 Chill Medicated,Edible,Beverage,nan,nan \n", "1 Feelz,Flower,Bud,Space Cowboy,3.5 \n", "2 Champelli,Flower,Bud,Xclusivo,3.5 \n", "3 CAM,Flower,Bud,Mellowz #7,7 \n", "4 Backpack Boyz,Edible,CBD Tincture/Caps/etc,nan... \n", "5 Raw Garden,Concentrate,Diamonds,Pink Lemonade,1 \n", "6 Jeeter,Preroll,Infused Joint,Peaches,12.5 \n", "7 Ocean Breeze Cultivators,Preroll,Blunt,Truffle... \n", "8 Cookies,Preroll,Joint,Triple Scoop,1 \n", "9 STIIIZY,Vape,Vape,Birthday Cake,1 \n", "10 Cookies,Preroll,Infused Joint,TangEray,1 \n", "11 Kanha,Edible,Gummies,nan,nan \n", "12 Fresca,Flower,Bud,BOBBI HYLL,3.5 \n", "13 Flav,Edible,Gummies,nan,nan \n", "14 Left Coast Extracts,Vape,510 cart,Cherry Pie,1 \n", "15 Minntz,Preroll,Joint,Emerald Cut,1 \n", "16 The Growers Circle,Flower,Bud,Double Down,3.5 \n", "17 Pacific Stone,Flower,Bud,nan,14 \n", "18 The Growers Circle,Preroll,Joint,Zoo Dawg x Co... \n", "19 Flight Bites,Edible,Gummies,nan,nan \n", "\n", " prediction \n", "0 Chill Medicated,Edible,Beverage,Watermelon,250 \n", "1 Feelz,Flower,Bud,Space Cowboy,3.5 \n", "2 Champelli,Flower,Bud,Xclusivo,3.5 \n", "3 CAM - Mellowz #7 7g,CAM,Flower,Bud,Mellowz,7 \n", "4 Backpack Boyz,Edible,Syrup,Bubblegum Gelato,1 \n", "5 Raw Garden,Concentrate,Diamonds,Pink Lemonade,1.0 \n", "6 Jeeter,Preroll,Infused Joint,Peaches,12.5 \n", "7 Ocean Breeze Cultivators,Blunt,Truffle Pupil,1.0 \n", "8 Cookies,Preroll,Joint,Triple Scoop,1.0 \n", "9 Birthday Cake Pod,STIIIZY,Vape,Birthday Cake,1.0 \n", "10 Here is the missing information:\\n\\nproduct_na... \n", "11 Kanha,Edible,Gummies,nan,nan \n", "12 Fresca,Flower,Bud,BOBBI HYLL,3.5 \n", "13 Flav,Edible,Gummies,Blueberry,100 \n", "14 Cherry Pie - 1g - Pod,Left Coast Extracts,Vape... \n", "15 Minntz,Preroll,Joint,Emerald Cut,1 \n", "16 The Growers Circle,Flower,Bud,Double Down,3.5 \n", "17 Pacific Stone,Flower,Bud,Blue Dream,14 \n", "18 The Grower Circle,Preroll,Joint,Zoo Dawg x Cos... \n", "19 Flight Bites,Edible,Gummies,S'mores,10 " ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "\n", "test_results = pd.DataFrame(\n", " zip(test_prompts,\n", " test_completions,\n", " [ele['choices'][0]['message']['content'] for ele in test_predictions])\n", " , columns = ['prompt','completion','prediction'])\n", "test_results" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'completion': ['Chill Medicated', 'Edible', 'Beverage', 'nan', 'nan'],\n", " 'prediction': ['Chill Medicated', 'Edible', 'Beverage', 'Watermelon', '250'],\n", " 'matches': [True, True, True, False, False],\n", " 'num_correct': 3}" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from util import compare_completion_and_prediction\n", "\n", "compare_completion_and_prediction(test_completions[0], test_predictions[0][\"choices\"][0][\"message\"][\"content\"])" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "66" ] }, "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Write function that uses compare_completion_and_prediction to return num_correct and return zero if there is an error\n", "def get_num_correct(completion, prediction):\n", " try:\n", " return compare_completion_and_prediction(completion, prediction)['num_correct']\n", " except:\n", " return 0 # this will be the case when format is incorrect\n", "\n", "# Apply get_num_correct function to test_results dataframe\n", "test_results['num_correct'] = test_results.apply(lambda row: get_num_correct(row['completion'], row['prediction']), axis=1)\n", "test_results['num_correct'].sum() # out of 100 possible correct predictions (20 samples * 5 cols per sample)" ] } ], "metadata": { "kernelspec": { "display_name": "kd-llm-dc", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.11" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }