Upload folder using huggingface_hub

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "cSpell.words": [
+        "openai",
+        "sqlalchemy",
+        "streamlit"
+    ]
+}

app/app.py

@@ -2,6 +2,7 @@
 
 import streamlit as st
 from openai_chat_completion import OpenAIChatCompletions
+from dataclean_hf import main
 
 st.title("Kaleidoscope Data - Data Cleaning LLM App")
 
@@ -9,22 +10,44 @@ st.write("This app is a demo of the LLM model for data cleaning. It is a work in
 
 # text box or csv upload
 text_input = st.text_input("Enter text", "")
-# csv_file = st.file_uploader("Upload CSV", type=['csv'])
+csv_file = st.file_uploader("Upload CSV", type=['csv'])
 
 # button to run data cleaning API on text via c class in openai_chat_completion.py
 if st.button("Run Data Cleaning API"):
     
     # if text_input is not empty, run data cleaning API on text_input
+    if text_input:
+
+        model = "gpt-4" # "gpt-3.5-turbo"
+        sys_mes = "prompts/gpt4-system-message.txt"
+        
+        # instantiate OpenAIChatCompletions class
+        # get response from openai_chat_completion method
+        chat = OpenAIChatCompletions(model=model, system_message=sys_mes)
+        response = chat.openai_chat_completion(text_input, n_shot=5)
+        
+        
+        # display response
+        st.write(response['choices'][0]['message']['content'])
     
-    model = "gpt-4" # "gpt-3.5-turbo"
-    sys_mes = "prompts/gpt4-system-message.txt"
-    
-    # instantiate OpenAIChatCompletions class
-    # get response from openai_chat_completion method
-    chat = OpenAIChatCompletions(model=model, system_message=sys_mes)
-    response = chat.openai_chat_completion(text_input, n_shot=5)
-    
-    
-    # display response
-    st.write(response['choices'][0]['message']['content'])
+    # if csv_file is not empty, run data cleaning API on csv_file
+    elif csv_file:
+        
+        # run data cleaning API on csv_file    
+        output_df = main(csv_file)
+
+        @st.cache
+        def convert_df(df):
+            # IMPORTANT: Cache the conversion to prevent computation on every rerun
+            return df.to_csv().encode('utf-8')
+
+        csv = convert_df(output_df)
+
+        st.download_button(
+            label="Download data as CSV",
+            data=csv,
+            file_name='cleaned_df.csv',
+            mime='text/csv',
+        )
+            
     

app/dataclean_hf.py

@@ -0,0 +1,151 @@
+import json
+from io import StringIO
+import csv
+import pandas as pd
+import numpy as np
+import streamlit as st
+
+from sqlalchemy import create_engine
+from yallmf.utils import run_with_timeout
+import openai
+
+from tqdm import tqdm
+
+
+openai.api_key = st.secrets["OPENAI_API_KEY"]
+
+def clean_data(
+    input_product_names: pd.Series, 
+    input_brands: pd.Series,
+    input_product_categories: pd.Series,
+    category_taxonomy: dict):
+
+    output_cols = ['brand', 'product_category', 'sub_product_category', 'strain_name']
+    ncols = len(output_cols)
+    
+    p1 = f'''
+    I am going to provide a data set of marijuana products and their metadata. Using the information I provide, I want you to provide me with the following information about the products.
+
+    - Brand (brand)
+    - product category (product_category)
+    - sub product category (sub_product_category)
+    - strain name (strain_name)
+
+    The following JSON shows all the acceptable Product Categories and their Sub Product Categories. Strictly adhere to the below mapping for valid product_category to sub_product_category relationships:
+
+    {json.dumps(category_taxonomy)}
+    
+    Additional requirements: 
+
+    - The input data set in CSV format, with commas as field delimiter and newline as row delimiter.
+    - Do not automatically assume that the information in the data set I provide is accurate.
+    - Leave the 'sub_product_category' field blank unless there's a clear and direct match with one of the categories provided in the list.If there is no explicit information to confidently assign a sub_product_category, default to leaving it blank.
+    - Strain names are only applicable for the following product categories: concentrate, preroll, vape, flower
+    - Look for clues in the product name to determine what brand/ product category/ sub product category/ and strain name the product should fall under. For Vape products, consider the words before 'Cartridge' or 'Cart' in the product name as potential strain names.
+    - Every row of the Output CSV must have EXACTLY {ncols} columns.
+    - When a field is left empty (e.g., 'sub_product_category' or 'strain_name'), simply leave it empty without placing an additional comma. Each row in the output CSV should always have only three commas separating the four fields regardless of whether some fields are empty. For instance, if 'sub_product_category' and 'strain_name' are empty, a row would look like this: "brand,product_category,,"
+    - DO NOT EXPLAIN YOURSELF, ONLY RETURN A CSV WITH THESE COLUMNS: {', '.join(output_cols)}
+
+    Input data set in CSV format:
+
+    '''
+    df = pd.DataFrame({'input__product_name':input_product_names,
+                    'input__brand':input_brands,
+                    'input__product_category':input_product_categories}).reset_index(drop=True)
+    # remove commas from all strings
+    df2 = df.copy()
+    for col in df2.columns:
+        df2[col] = df2[col].str.replace(',', '')
+    
+    # send to LLM
+    p2 = df2.to_csv(index=False, quoting=csv.QUOTE_ALL)
+    messages = [{'role':'system','content':'You are a helpful assistant. Return a properly-formatted CSV with the correct number of columns.'},
+                {'role':'user', 'content':p1+p2+'\n\nOutput CSV with header row:\n\n'}
+                ]
+    comp = run_with_timeout(openai.ChatCompletion.create,
+        model='gpt-4',
+        messages=messages,
+        max_tokens=2000,
+        timeout=300,
+        temperature=0.2
+        )
+    res = comp['choices'][0]['message']['content']
+
+    # remove rows with wrong number of columns
+    keeprows = []
+    for i,s in enumerate(res.split('\n')):
+        if i==0:
+            keeprows.append(s)
+            continue
+        _ncols = len(s.split(','))
+        if _ncols!=ncols:
+            print(f'Got {_ncols} columns, skipping row {i-1} ({s})')
+            df = df.drop(i-1)
+        else:
+            keeprows.append(s)
+    df = df.reset_index(drop=True)
+
+    resdf = pd.read_csv(StringIO('\n'.join(keeprows)))
+
+    assert len(df)==len(resdf), 'Result CSV did not match input CSV in length'
+    df = pd.concat([df.reset_index(drop=True),resdf.reset_index(drop=True)],axis=1)
+    # check category/subcategory
+    dropidxs=[]
+    for idx, row in df.iterrows():
+        drop = False
+        if pd.isna(row['product_category']) and not pd.isna(row['sub_product_category']):
+            drop=True
+            print('product_category is null while sub_product_category is not null, dropping')
+        if not pd.isna(row['product_category']):
+            if row['product_category'] not in category_taxonomy.keys():
+                print(f'category "{row["product_category"]}" not in taxonomy, dropping row')
+                drop =True
+            elif not pd.isna(row['sub_product_category']):
+                if row['sub_product_category'] not in category_taxonomy[row['product_category']]:
+                    print(f'subcategory "{row["sub_product_category"]}" not valid for category {row["product_category"]}, dropping row')
+                    drop =True
+        if drop:
+            dropidxs.append(idx)
+    df = df.drop(dropidxs)
+
+    return df
+
+def get_key(df):
+    return df['input__product_name'] + df['input__brand'] + df['input__product_category']
+
+def main(upload_df: pd.DataFrame,
+        chunksize: int = 30,
+        ):
+    category_taxonomy = {
+        "Wellness": ["Mushroom Caps", "CBD Tincture/Caps/etc", "Promo/ Sample", "Capsule", "Liquid Flower", ""],
+        "Concentrate": ["Diamonds", "Shatter", "Sugar", "Promo/ Sample", "Badder", "Diamonds and Sauce", "Rosin", "Cookies Dough", "Flan", "Cookie Dough", ""],
+        "Preroll": ["Cubano", "Joint", "Promo/ Sample", "Blunt", "Infused Joint", "Packwoods Blunt", "Infused Blunt", "Napalm", ""],
+        "Vape": ["Terp Sauce", "Gpen 0.5", "Cured Resin", "Solventless Rosin", "510", "Dry Flower Series", "Natural Terp Series", "Promo/ Sample", "Dart Pod 0.5", "Raw Garden", "Live Flower Series", "Rosin", "Disposable", ""],
+        "Edible": ["Cookies", "Gummies", "Mint", "Promo/ Sample", "Beverage", "Chocolate", ""],
+        "Grow Products": ["Promo/ Sample", ""],
+        "Flower": ["Promo/ Sample", "Bud", ""],
+        "Accessory": ["Promo/ Sample", ""]
+    }
+
+    # join together and get the diff
+    upload_df['key'] = get_key(upload_df)
+    upload_df=upload_df.set_index('key')
+    if output_df is None:
+        rundf = upload_df
+        outlen = 0
+    else:
+        output_df['key'] = get_key(output_df)
+        output_df=output_df.set_index('key')
+        rundf = upload_df.loc[~upload_df.index.isin(output_df.index)]
+        outlen = len(output_df)
+
+
+    # st.write(f'Input size: {len(upload_df)}, Output size: {outlen}, Still to process: {len(rundf)}')
+
+    for _, chunk in tqdm(rundf.groupby(np.arange(len(rundf)) // chunksize)):
+        result = clean_data(chunk['input__product_name'], chunk['input__brand'], chunk['input__product_category'], category_taxonomy)
+        result['key'] = get_key(result)
+        result = result.set_index('key')
+        output_df = result if output_df is None else pd.concat([output_df, result])
+    
+    return output_df