cmagganas commited on
Commit
03e5095
1 Parent(s): b96ed04

Delete app/cookies_llm_experimental_eval.ipynb

Browse files
app/cookies_llm_experimental_eval.ipynb DELETED
@@ -1,412 +0,0 @@
1
- {
2
- "cells": [
3
- {
4
- "attachments": {},
5
- "cell_type": "markdown",
6
- "metadata": {},
7
- "source": [
8
- "## This notebook is used to evaluate the performance of the LLM model to clean the cookies dataset."
9
- ]
10
- },
11
- {
12
- "attachments": {},
13
- "cell_type": "markdown",
14
- "metadata": {},
15
- "source": [
16
- "#### Using OpenAI API"
17
- ]
18
- },
19
- {
20
- "attachments": {},
21
- "cell_type": "markdown",
22
- "metadata": {},
23
- "source": [
24
- "Try to call function from newly created module `openai_chat_completion.py`"
25
- ]
26
- },
27
- {
28
- "cell_type": "code",
29
- "execution_count": 3,
30
- "metadata": {},
31
- "outputs": [
32
- {
33
- "name": "stdout",
34
- "output_type": "stream",
35
- "text": [
36
- "{\n",
37
- " \"id\": \"chatcmpl-7UecmbvjkbwamEpYXyIvGO4w4Ae8i\",\n",
38
- " \"object\": \"chat.completion\",\n",
39
- " \"created\": 1687540356,\n",
40
- " \"model\": \"gpt-4-0314\",\n",
41
- " \"choices\": [\n",
42
- " {\n",
43
- " \"index\": 0,\n",
44
- " \"message\": {\n",
45
- " \"role\": \"assistant\",\n",
46
- " \"content\": \"Chill Medicated,Edible,Beverage,Watermelon,250\"\n",
47
- " },\n",
48
- " \"finish_reason\": \"stop\"\n",
49
- " }\n",
50
- " ],\n",
51
- " \"usage\": {\n",
52
- " \"prompt_tokens\": 636,\n",
53
- " \"completion_tokens\": 15,\n",
54
- " \"total_tokens\": 651\n",
55
- " }\n",
56
- "}\n"
57
- ]
58
- }
59
- ],
60
- "source": [
61
- "import os\n",
62
- "from dotenv import load_dotenv\n",
63
- "load_dotenv()\n",
64
- "\n",
65
- "import openai\n",
66
- "\n",
67
- "# set OPENAI_API_KEY environment variable from .env file\n",
68
- "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
69
- "\n",
70
- "# import OpenAIChatCompletions class from openai_chat_completion.py file located in llm_data_cleaner/scripts folder (this notebook is located in llm_data_cleaner/notebooks folder)\n",
71
- "from openai_chat_completion import OpenAIChatCompletions, compare_completion_and_prediction\n",
72
- "\n",
73
- "# read in llm-data-cleaner/prompts/gpt4-system-message.txt file into variable system_message\n",
74
- "system_message = open('../prompts/gpt4-system-message.txt', 'r').read()\n",
75
- "\n",
76
- "# create an instance of the OpenAIChatCompletions class and use the openai_chat_completion method to get chat completions\n",
77
- "chat_completions = OpenAIChatCompletions(model=\"gpt-4\", system_message=system_message)\n",
78
- "\n",
79
- "prompt = \"co-2MFE5QVF,Chill Medicated - Watermelon - Syrup - 250mg,Chill Medicated,nan,nan,nan\"\n",
80
- "completion = chat_completions.openai_chat_completion(prompt, n_shot=1)\n",
81
- "print(completion)"
82
- ]
83
- },
84
- {
85
- "cell_type": "code",
86
- "execution_count": 7,
87
- "metadata": {},
88
- "outputs": [],
89
- "source": [
90
- "from openai_chat_completion import OpenAIChatCompletions\n",
91
- "\n",
92
- "test_prompts, test_completions, test_predictions = chat_completions.predict_jsonl(n_shot=1)"
93
- ]
94
- },
95
- {
96
- "cell_type": "code",
97
- "execution_count": 44,
98
- "metadata": {},
99
- "outputs": [
100
- {
101
- "data": {
102
- "text/html": [
103
- "<div>\n",
104
- "<style scoped>\n",
105
- " .dataframe tbody tr th:only-of-type {\n",
106
- " vertical-align: middle;\n",
107
- " }\n",
108
- "\n",
109
- " .dataframe tbody tr th {\n",
110
- " vertical-align: top;\n",
111
- " }\n",
112
- "\n",
113
- " .dataframe thead th {\n",
114
- " text-align: right;\n",
115
- " }\n",
116
- "</style>\n",
117
- "<table border=\"1\" class=\"dataframe\">\n",
118
- " <thead>\n",
119
- " <tr style=\"text-align: right;\">\n",
120
- " <th></th>\n",
121
- " <th>prompt</th>\n",
122
- " <th>completion</th>\n",
123
- " <th>prediction</th>\n",
124
- " </tr>\n",
125
- " </thead>\n",
126
- " <tbody>\n",
127
- " <tr>\n",
128
- " <th>0</th>\n",
129
- " <td>co-2MFE5QVF,Chill Medicated - Watermelon - Syr...</td>\n",
130
- " <td>Chill Medicated,Edible,Beverage,nan,nan</td>\n",
131
- " <td>Chill Medicated,Edible,Beverage,Watermelon,250</td>\n",
132
- " </tr>\n",
133
- " <tr>\n",
134
- " <th>1</th>\n",
135
- " <td>bl-111630024545,Feelz - Space Cowboy 3.5g,nan,...</td>\n",
136
- " <td>Feelz,Flower,Bud,Space Cowboy,3.5</td>\n",
137
- " <td>Feelz,Flower,Bud,Space Cowboy,3.5</td>\n",
138
- " </tr>\n",
139
- " <tr>\n",
140
- " <th>2</th>\n",
141
- " <td>fl-8voAjt83sD,Champelli | Xclusivo 3.5g | Eigh...</td>\n",
142
- " <td>Champelli,Flower,Bud,Xclusivo,3.5</td>\n",
143
- " <td>Champelli,Flower,Bud,Xclusivo,3.5</td>\n",
144
- " </tr>\n",
145
- " <tr>\n",
146
- " <th>3</th>\n",
147
- " <td>bl-073133213364,CAM - Mellowz #7 7g,nan,FLOWER...</td>\n",
148
- " <td>CAM,Flower,Bud,Mellowz #7,7</td>\n",
149
- " <td>CAM - Mellowz #7 7g,CAM,Flower,Bud,Mellowz,7</td>\n",
150
- " </tr>\n",
151
- " <tr>\n",
152
- " <th>4</th>\n",
153
- " <td>fl-fwJQL2AWnS,Backpack Boyz | Bubblegum Gelato...</td>\n",
154
- " <td>Backpack Boyz,Edible,CBD Tincture/Caps/etc,nan...</td>\n",
155
- " <td>Backpack Boyz,Edible,Syrup,Bubblegum Gelato,1</td>\n",
156
- " </tr>\n",
157
- " <tr>\n",
158
- " <th>5</th>\n",
159
- " <td>fl-dXE5cH45AQ,Raw Garden | Pink Lemonade Crush...</td>\n",
160
- " <td>Raw Garden,Concentrate,Diamonds,Pink Lemonade,1</td>\n",
161
- " <td>Raw Garden,Concentrate,Diamonds,Pink Lemonade,1.0</td>\n",
162
- " </tr>\n",
163
- " <tr>\n",
164
- " <th>6</th>\n",
165
- " <td>md-1159983,Baby Jeeter Peaches | Infused Prero...</td>\n",
166
- " <td>Jeeter,Preroll,Infused Joint,Peaches,12.5</td>\n",
167
- " <td>Jeeter,Preroll,Infused Joint,Peaches,12.5</td>\n",
168
- " </tr>\n",
169
- " <tr>\n",
170
- " <th>7</th>\n",
171
- " <td>co-6WGV1Z0H,Ocean Breeze Cultivators - Truffle...</td>\n",
172
- " <td>Ocean Breeze Cultivators,Preroll,Blunt,Truffle...</td>\n",
173
- " <td>Ocean Breeze Cultivators,Blunt,Truffle Pupil,1.0</td>\n",
174
- " </tr>\n",
175
- " <tr>\n",
176
- " <th>8</th>\n",
177
- " <td>fl-doWkMzvFq2,Cookies | Triple Scoop Preroll 1...</td>\n",
178
- " <td>Cookies,Preroll,Joint,Triple Scoop,1</td>\n",
179
- " <td>Cookies,Preroll,Joint,Triple Scoop,1.0</td>\n",
180
- " </tr>\n",
181
- " <tr>\n",
182
- " <th>9</th>\n",
183
- " <td>bl-842922110296,STIIIZY - Birthday Cake Pod 1g...</td>\n",
184
- " <td>STIIIZY,Vape,Vape,Birthday Cake,1</td>\n",
185
- " <td>Birthday Cake Pod,STIIIZY,Vape,Birthday Cake,1.0</td>\n",
186
- " </tr>\n",
187
- " <tr>\n",
188
- " <th>10</th>\n",
189
- " <td>co-6GGKA0GV,Cookies x The Grower Circle - Tang...</td>\n",
190
- " <td>Cookies,Preroll,Infused Joint,TangEray,1</td>\n",
191
- " <td>Here is the missing information:\\n\\nproduct_na...</td>\n",
192
- " </tr>\n",
193
- " <tr>\n",
194
- " <th>11</th>\n",
195
- " <td>co-7YGGNHBR,Kanha - Nano Blood Orange Bliss In...</td>\n",
196
- " <td>Kanha,Edible,Gummies,nan,nan</td>\n",
197
- " <td>Kanha,Edible,Gummies,nan,nan</td>\n",
198
- " </tr>\n",
199
- " <tr>\n",
200
- " <th>12</th>\n",
201
- " <td>md-1242517,BOBBI HYLL | Indoor - 3.5g,Fresca,F...</td>\n",
202
- " <td>Fresca,Flower,Bud,BOBBI HYLL,3.5</td>\n",
203
- " <td>Fresca,Flower,Bud,BOBBI HYLL,3.5</td>\n",
204
- " </tr>\n",
205
- " <tr>\n",
206
- " <th>13</th>\n",
207
- " <td>fl-9nEPGsnYtY,Flav | Blueberry Belts 100mg,Fla...</td>\n",
208
- " <td>Flav,Edible,Gummies,nan,nan</td>\n",
209
- " <td>Flav,Edible,Gummies,Blueberry,100</td>\n",
210
- " </tr>\n",
211
- " <tr>\n",
212
- " <th>14</th>\n",
213
- " <td>md-1347497,Cherry Pie - 1g - Pod,Left Coast Ex...</td>\n",
214
- " <td>Left Coast Extracts,Vape,510 cart,Cherry Pie,1</td>\n",
215
- " <td>Cherry Pie - 1g - Pod,Left Coast Extracts,Vape...</td>\n",
216
- " </tr>\n",
217
- " <tr>\n",
218
- " <th>15</th>\n",
219
- " <td>co-76GP441T,Minntz - Emerald Cut - Indoor - Jo...</td>\n",
220
- " <td>Minntz,Preroll,Joint,Emerald Cut,1</td>\n",
221
- " <td>Minntz,Preroll,Joint,Emerald Cut,1</td>\n",
222
- " </tr>\n",
223
- " <tr>\n",
224
- " <th>16</th>\n",
225
- " <td>co-5RAWYHYQ,The Growers Circle - Double Down -...</td>\n",
226
- " <td>The Growers Circle,Flower,Bud,Double Down,3.5</td>\n",
227
- " <td>The Growers Circle,Flower,Bud,Double Down,3.5</td>\n",
228
- " </tr>\n",
229
- " <tr>\n",
230
- " <th>17</th>\n",
231
- " <td>md-1195389,Blue Dream Roll Your Own Sugar Shak...</td>\n",
232
- " <td>Pacific Stone,Flower,Bud,nan,14</td>\n",
233
- " <td>Pacific Stone,Flower,Bud,Blue Dream,14</td>\n",
234
- " </tr>\n",
235
- " <tr>\n",
236
- " <th>18</th>\n",
237
- " <td>co-847ZXF37,The Grower Circle - Zoo Dawg x Cos...</td>\n",
238
- " <td>The Growers Circle,Preroll,Joint,Zoo Dawg x Co...</td>\n",
239
- " <td>The Grower Circle,Preroll,Joint,Zoo Dawg x Cos...</td>\n",
240
- " </tr>\n",
241
- " <tr>\n",
242
- " <th>19</th>\n",
243
- " <td>co-8EMW15ZM,Flight Bites - S'mores - Gummy - 1...</td>\n",
244
- " <td>Flight Bites,Edible,Gummies,nan,nan</td>\n",
245
- " <td>Flight Bites,Edible,Gummies,S'mores,10</td>\n",
246
- " </tr>\n",
247
- " </tbody>\n",
248
- "</table>\n",
249
- "</div>"
250
- ],
251
- "text/plain": [
252
- " prompt \\\n",
253
- "0 co-2MFE5QVF,Chill Medicated - Watermelon - Syr... \n",
254
- "1 bl-111630024545,Feelz - Space Cowboy 3.5g,nan,... \n",
255
- "2 fl-8voAjt83sD,Champelli | Xclusivo 3.5g | Eigh... \n",
256
- "3 bl-073133213364,CAM - Mellowz #7 7g,nan,FLOWER... \n",
257
- "4 fl-fwJQL2AWnS,Backpack Boyz | Bubblegum Gelato... \n",
258
- "5 fl-dXE5cH45AQ,Raw Garden | Pink Lemonade Crush... \n",
259
- "6 md-1159983,Baby Jeeter Peaches | Infused Prero... \n",
260
- "7 co-6WGV1Z0H,Ocean Breeze Cultivators - Truffle... \n",
261
- "8 fl-doWkMzvFq2,Cookies | Triple Scoop Preroll 1... \n",
262
- "9 bl-842922110296,STIIIZY - Birthday Cake Pod 1g... \n",
263
- "10 co-6GGKA0GV,Cookies x The Grower Circle - Tang... \n",
264
- "11 co-7YGGNHBR,Kanha - Nano Blood Orange Bliss In... \n",
265
- "12 md-1242517,BOBBI HYLL | Indoor - 3.5g,Fresca,F... \n",
266
- "13 fl-9nEPGsnYtY,Flav | Blueberry Belts 100mg,Fla... \n",
267
- "14 md-1347497,Cherry Pie - 1g - Pod,Left Coast Ex... \n",
268
- "15 co-76GP441T,Minntz - Emerald Cut - Indoor - Jo... \n",
269
- "16 co-5RAWYHYQ,The Growers Circle - Double Down -... \n",
270
- "17 md-1195389,Blue Dream Roll Your Own Sugar Shak... \n",
271
- "18 co-847ZXF37,The Grower Circle - Zoo Dawg x Cos... \n",
272
- "19 co-8EMW15ZM,Flight Bites - S'mores - Gummy - 1... \n",
273
- "\n",
274
- " completion \\\n",
275
- "0 Chill Medicated,Edible,Beverage,nan,nan \n",
276
- "1 Feelz,Flower,Bud,Space Cowboy,3.5 \n",
277
- "2 Champelli,Flower,Bud,Xclusivo,3.5 \n",
278
- "3 CAM,Flower,Bud,Mellowz #7,7 \n",
279
- "4 Backpack Boyz,Edible,CBD Tincture/Caps/etc,nan... \n",
280
- "5 Raw Garden,Concentrate,Diamonds,Pink Lemonade,1 \n",
281
- "6 Jeeter,Preroll,Infused Joint,Peaches,12.5 \n",
282
- "7 Ocean Breeze Cultivators,Preroll,Blunt,Truffle... \n",
283
- "8 Cookies,Preroll,Joint,Triple Scoop,1 \n",
284
- "9 STIIIZY,Vape,Vape,Birthday Cake,1 \n",
285
- "10 Cookies,Preroll,Infused Joint,TangEray,1 \n",
286
- "11 Kanha,Edible,Gummies,nan,nan \n",
287
- "12 Fresca,Flower,Bud,BOBBI HYLL,3.5 \n",
288
- "13 Flav,Edible,Gummies,nan,nan \n",
289
- "14 Left Coast Extracts,Vape,510 cart,Cherry Pie,1 \n",
290
- "15 Minntz,Preroll,Joint,Emerald Cut,1 \n",
291
- "16 The Growers Circle,Flower,Bud,Double Down,3.5 \n",
292
- "17 Pacific Stone,Flower,Bud,nan,14 \n",
293
- "18 The Growers Circle,Preroll,Joint,Zoo Dawg x Co... \n",
294
- "19 Flight Bites,Edible,Gummies,nan,nan \n",
295
- "\n",
296
- " prediction \n",
297
- "0 Chill Medicated,Edible,Beverage,Watermelon,250 \n",
298
- "1 Feelz,Flower,Bud,Space Cowboy,3.5 \n",
299
- "2 Champelli,Flower,Bud,Xclusivo,3.5 \n",
300
- "3 CAM - Mellowz #7 7g,CAM,Flower,Bud,Mellowz,7 \n",
301
- "4 Backpack Boyz,Edible,Syrup,Bubblegum Gelato,1 \n",
302
- "5 Raw Garden,Concentrate,Diamonds,Pink Lemonade,1.0 \n",
303
- "6 Jeeter,Preroll,Infused Joint,Peaches,12.5 \n",
304
- "7 Ocean Breeze Cultivators,Blunt,Truffle Pupil,1.0 \n",
305
- "8 Cookies,Preroll,Joint,Triple Scoop,1.0 \n",
306
- "9 Birthday Cake Pod,STIIIZY,Vape,Birthday Cake,1.0 \n",
307
- "10 Here is the missing information:\\n\\nproduct_na... \n",
308
- "11 Kanha,Edible,Gummies,nan,nan \n",
309
- "12 Fresca,Flower,Bud,BOBBI HYLL,3.5 \n",
310
- "13 Flav,Edible,Gummies,Blueberry,100 \n",
311
- "14 Cherry Pie - 1g - Pod,Left Coast Extracts,Vape... \n",
312
- "15 Minntz,Preroll,Joint,Emerald Cut,1 \n",
313
- "16 The Growers Circle,Flower,Bud,Double Down,3.5 \n",
314
- "17 Pacific Stone,Flower,Bud,Blue Dream,14 \n",
315
- "18 The Grower Circle,Preroll,Joint,Zoo Dawg x Cos... \n",
316
- "19 Flight Bites,Edible,Gummies,S'mores,10 "
317
- ]
318
- },
319
- "execution_count": 44,
320
- "metadata": {},
321
- "output_type": "execute_result"
322
- }
323
- ],
324
- "source": [
325
- "import pandas as pd\n",
326
- "\n",
327
- "test_results = pd.DataFrame(\n",
328
- " zip(test_prompts,\n",
329
- " test_completions,\n",
330
- " [ele['choices'][0]['message']['content'] for ele in test_predictions])\n",
331
- " , columns = ['prompt','completion','prediction'])\n",
332
- "test_results"
333
- ]
334
- },
335
- {
336
- "cell_type": "code",
337
- "execution_count": 27,
338
- "metadata": {},
339
- "outputs": [
340
- {
341
- "data": {
342
- "text/plain": [
343
- "{'completion': ['Chill Medicated', 'Edible', 'Beverage', 'nan', 'nan'],\n",
344
- " 'prediction': ['Chill Medicated', 'Edible', 'Beverage', 'Watermelon', '250'],\n",
345
- " 'matches': [True, True, True, False, False],\n",
346
- " 'num_correct': 3}"
347
- ]
348
- },
349
- "execution_count": 27,
350
- "metadata": {},
351
- "output_type": "execute_result"
352
- }
353
- ],
354
- "source": [
355
- "from util import compare_completion_and_prediction\n",
356
- "\n",
357
- "compare_completion_and_prediction(test_completions[0], test_predictions[0][\"choices\"][0][\"message\"][\"content\"])"
358
- ]
359
- },
360
- {
361
- "cell_type": "code",
362
- "execution_count": 58,
363
- "metadata": {},
364
- "outputs": [
365
- {
366
- "data": {
367
- "text/plain": [
368
- "66"
369
- ]
370
- },
371
- "execution_count": 58,
372
- "metadata": {},
373
- "output_type": "execute_result"
374
- }
375
- ],
376
- "source": [
377
- "# Write function that uses compare_completion_and_prediction to return num_correct and return zero if there is an error\n",
378
- "def get_num_correct(completion, prediction):\n",
379
- " try:\n",
380
- " return compare_completion_and_prediction(completion, prediction)['num_correct']\n",
381
- " except:\n",
382
- " return 0 # this will be the case when format is incorrect\n",
383
- "\n",
384
- "# Apply get_num_correct function to test_results dataframe\n",
385
- "test_results['num_correct'] = test_results.apply(lambda row: get_num_correct(row['completion'], row['prediction']), axis=1)\n",
386
- "test_results['num_correct'].sum() # out of 100 possible correct predictions (20 samples * 5 cols per sample)"
387
- ]
388
- }
389
- ],
390
- "metadata": {
391
- "kernelspec": {
392
- "display_name": "kd-llm-dc",
393
- "language": "python",
394
- "name": "python3"
395
- },
396
- "language_info": {
397
- "codemirror_mode": {
398
- "name": "ipython",
399
- "version": 3
400
- },
401
- "file_extension": ".py",
402
- "mimetype": "text/x-python",
403
- "name": "python",
404
- "nbconvert_exporter": "python",
405
- "pygments_lexer": "ipython3",
406
- "version": "3.10.11"
407
- },
408
- "orig_nbformat": 4
409
- },
410
- "nbformat": 4,
411
- "nbformat_minor": 2
412
- }