assets/notebooks/seal_imdb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', 0)\n",
    "import numpy as np\n",
    "import torch\n",
    "import math\n",
    "import openai\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoTokenizer\n",
    "import nltk\n",
    "from nltk.cluster import KMeansClusterer\n",
    "import scipy.spatial.distance as sdist\n",
    "from scipy.spatial import distance_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "from datasets import load_dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../seal/')\n",
    "from run_inference import run_inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'imdb'\n",
    "model = 'lvwerra/distilbert-imdb'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-09 21:41:20.800 WARNING datasets.builder: Reusing dataset imdb (/Users/nazneenrajani/.cache/huggingface/datasets/imdb/plain_text/1.0.0/2fdd8b9bcadd6e7055e742a706876ba43f19faee861df134affd7a3f60fc38a1)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Connection error, and we cannot find the requested files in the cached path. Please try again or make sure your Internet connection is on.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m/Users/nazneenrajani/workspace/spaces/seal/assets/notebooks/seal_imdb.ipynb Cell 5\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/nazneenrajani/workspace/spaces/seal/assets/notebooks/seal_imdb.ipynb#ch0000024?line=0'>1</a>\u001b[0m run_inference(dataset\u001b[39m=\u001b[39;49mdataset,model\u001b[39m=\u001b[39;49mmodel,split\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39mtest\u001b[39;49m\u001b[39m'\u001b[39;49m)\n",
      "File \u001b[0;32m~/workspace/spaces/seal/assets/notebooks/../../seal/run_inference.py:38\u001b[0m, in \u001b[0;36mrun_inference\u001b[0;34m(dataset, model, split, output_path)\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_inference\u001b[39m(dataset\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39myelp_polarity\u001b[39m\u001b[39m'\u001b[39m, model\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mtextattack/albert-base-v2-yelp-polarity\u001b[39m\u001b[39m'\u001b[39m, split\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mtest\u001b[39m\u001b[39m'\u001b[39m, output_path\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m./assets/data/inference_results\u001b[39m\u001b[39m'\u001b[39m):\n\u001b[0;32m---> 38\u001b[0m     tokenizer, dataloader \u001b[39m=\u001b[39m load_session(dataset,model,split)\n\u001b[1;32m     39\u001b[0m     hook, hidden_layers \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mclassifier\u001b[39m.\u001b[39mregister_forward_hook(get_input(\u001b[39m'\u001b[39m\u001b[39mlast_layer\u001b[39m\u001b[39m'\u001b[39m))\n\u001b[1;32m     40\u001b[0m     \u001b[39m# Run inference on entire dataset\u001b[39;00m\n",
      "File \u001b[0;32m~/workspace/spaces/seal/assets/notebooks/../../seal/run_inference.py:24\u001b[0m, in \u001b[0;36mload_session\u001b[0;34m(dataset, model, split)\u001b[0m\n\u001b[1;32m     18\u001b[0m dataloader \u001b[39m=\u001b[39m DataLoader(\n\u001b[1;32m     19\u001b[0m     dataset,\n\u001b[1;32m     20\u001b[0m     batch_size\u001b[39m=\u001b[39m\u001b[39m256\u001b[39m, drop_last\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m\n\u001b[1;32m     21\u001b[0m )\n\u001b[1;32m     23\u001b[0m model \u001b[39m=\u001b[39m AutoModelForSequenceClassification\u001b[39m.\u001b[39mfrom_pretrained(model)\n\u001b[0;32m---> 24\u001b[0m tokenizer \u001b[39m=\u001b[39m AutoTokenizer\u001b[39m.\u001b[39;49mfrom_pretrained(model)\n\u001b[1;32m     25\u001b[0m model\u001b[39m.\u001b[39meval()\n\u001b[1;32m     26\u001b[0m model\u001b[39m.\u001b[39mto(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/envs/autoeval/lib/python3.9/site-packages/transformers/models/auto/tokenization_auto.py:471\u001b[0m, in \u001b[0;36mAutoTokenizer.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m    468\u001b[0m     \u001b[39mreturn\u001b[39;00m tokenizer_class\u001b[39m.\u001b[39mfrom_pretrained(pretrained_model_name_or_path, \u001b[39m*\u001b[39minputs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    470\u001b[0m \u001b[39m# Next, let's try to use the tokenizer_config file to get the tokenizer class.\u001b[39;00m\n\u001b[0;32m--> 471\u001b[0m tokenizer_config \u001b[39m=\u001b[39m get_tokenizer_config(pretrained_model_name_or_path, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    472\u001b[0m config_tokenizer_class \u001b[39m=\u001b[39m tokenizer_config\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mtokenizer_class\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    473\u001b[0m tokenizer_auto_map \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/autoeval/lib/python3.9/site-packages/transformers/models/auto/tokenization_auto.py:332\u001b[0m, in \u001b[0;36mget_tokenizer_config\u001b[0;34m(pretrained_model_name_or_path, cache_dir, force_download, resume_download, proxies, use_auth_token, revision, local_files_only, **kwargs)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_tokenizer_config\u001b[39m(\n\u001b[1;32m    264\u001b[0m     pretrained_model_name_or_path: Union[\u001b[39mstr\u001b[39m, os\u001b[39m.\u001b[39mPathLike],\n\u001b[1;32m    265\u001b[0m     cache_dir: Optional[Union[\u001b[39mstr\u001b[39m, os\u001b[39m.\u001b[39mPathLike]] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    272\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs,\n\u001b[1;32m    273\u001b[0m ):\n\u001b[1;32m    274\u001b[0m     \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    275\u001b[0m \u001b[39m    Loads the tokenizer configuration from a pretrained model tokenizer configuration.\u001b[39;00m\n\u001b[1;32m    276\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    330\u001b[0m \u001b[39m    tokenizer_config = get_tokenizer_config(\"tokenizer-test\")\u001b[39;00m\n\u001b[1;32m    331\u001b[0m \u001b[39m    ```\"\"\"\u001b[39;00m\n\u001b[0;32m--> 332\u001b[0m     resolved_config_file \u001b[39m=\u001b[39m get_file_from_repo(\n\u001b[1;32m    333\u001b[0m         pretrained_model_name_or_path,\n\u001b[1;32m    334\u001b[0m         TOKENIZER_CONFIG_FILE,\n\u001b[1;32m    335\u001b[0m         cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[1;32m    336\u001b[0m         force_download\u001b[39m=\u001b[39;49mforce_download,\n\u001b[1;32m    337\u001b[0m         resume_download\u001b[39m=\u001b[39;49mresume_download,\n\u001b[1;32m    338\u001b[0m         proxies\u001b[39m=\u001b[39;49mproxies,\n\u001b[1;32m    339\u001b[0m         use_auth_token\u001b[39m=\u001b[39;49muse_auth_token,\n\u001b[1;32m    340\u001b[0m         revision\u001b[39m=\u001b[39;49mrevision,\n\u001b[1;32m    341\u001b[0m         local_files_only\u001b[39m=\u001b[39;49mlocal_files_only,\n\u001b[1;32m    342\u001b[0m     )\n\u001b[1;32m    343\u001b[0m     \u001b[39mif\u001b[39;00m resolved_config_file \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    344\u001b[0m         logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mCould not locate the tokenizer configuration file, will try to use the model config instead.\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/envs/autoeval/lib/python3.9/site-packages/transformers/utils/hub.py:678\u001b[0m, in \u001b[0;36mget_file_from_repo\u001b[0;34m(path_or_repo, filename, cache_dir, force_download, resume_download, proxies, use_auth_token, revision, local_files_only)\u001b[0m\n\u001b[1;32m    674\u001b[0m     resolved_file \u001b[39m=\u001b[39m hf_bucket_url(path_or_repo, filename\u001b[39m=\u001b[39mfilename, revision\u001b[39m=\u001b[39mrevision, mirror\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m)\n\u001b[1;32m    676\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    677\u001b[0m     \u001b[39m# Load from URL or cache if already cached\u001b[39;00m\n\u001b[0;32m--> 678\u001b[0m     resolved_file \u001b[39m=\u001b[39m cached_path(\n\u001b[1;32m    679\u001b[0m         resolved_file,\n\u001b[1;32m    680\u001b[0m         cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[1;32m    681\u001b[0m         force_download\u001b[39m=\u001b[39;49mforce_download,\n\u001b[1;32m    682\u001b[0m         proxies\u001b[39m=\u001b[39;49mproxies,\n\u001b[1;32m    683\u001b[0m         resume_download\u001b[39m=\u001b[39;49mresume_download,\n\u001b[1;32m    684\u001b[0m         local_files_only\u001b[39m=\u001b[39;49mlocal_files_only,\n\u001b[1;32m    685\u001b[0m         use_auth_token\u001b[39m=\u001b[39;49muse_auth_token,\n\u001b[1;32m    686\u001b[0m     )\n\u001b[1;32m    688\u001b[0m \u001b[39mexcept\u001b[39;00m RepositoryNotFoundError:\n\u001b[1;32m    689\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mEnvironmentError\u001b[39;00m(\n\u001b[1;32m    690\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mpath_or_repo\u001b[39m}\u001b[39;00m\u001b[39m is not a local folder and is not a valid model identifier \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    691\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mlisted on \u001b[39m\u001b[39m'\u001b[39m\u001b[39mhttps://huggingface.co/models\u001b[39m\u001b[39m'\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mIf this is a private repository, make sure to \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    692\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mpass a token having permission to this repo with `use_auth_token` or log in with \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    693\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39m`huggingface-cli login` and pass `use_auth_token=True`.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    694\u001b[0m     )\n",
      "File \u001b[0;32m~/miniconda3/envs/autoeval/lib/python3.9/site-packages/transformers/utils/hub.py:282\u001b[0m, in \u001b[0;36mcached_path\u001b[0;34m(url_or_filename, cache_dir, force_download, proxies, resume_download, user_agent, extract_compressed_file, force_extract, use_auth_token, local_files_only)\u001b[0m\n\u001b[1;32m    278\u001b[0m     local_files_only \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m    280\u001b[0m \u001b[39mif\u001b[39;00m is_remote_url(url_or_filename):\n\u001b[1;32m    281\u001b[0m     \u001b[39m# URL, so get it from the cache (downloading if necessary)\u001b[39;00m\n\u001b[0;32m--> 282\u001b[0m     output_path \u001b[39m=\u001b[39m get_from_cache(\n\u001b[1;32m    283\u001b[0m         url_or_filename,\n\u001b[1;32m    284\u001b[0m         cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[1;32m    285\u001b[0m         force_download\u001b[39m=\u001b[39;49mforce_download,\n\u001b[1;32m    286\u001b[0m         proxies\u001b[39m=\u001b[39;49mproxies,\n\u001b[1;32m    287\u001b[0m         resume_download\u001b[39m=\u001b[39;49mresume_download,\n\u001b[1;32m    288\u001b[0m         user_agent\u001b[39m=\u001b[39;49muser_agent,\n\u001b[1;32m    289\u001b[0m         use_auth_token\u001b[39m=\u001b[39;49muse_auth_token,\n\u001b[1;32m    290\u001b[0m         local_files_only\u001b[39m=\u001b[39;49mlocal_files_only,\n\u001b[1;32m    291\u001b[0m     )\n\u001b[1;32m    292\u001b[0m \u001b[39melif\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mexists(url_or_filename):\n\u001b[1;32m    293\u001b[0m     \u001b[39m# File, and it exists.\u001b[39;00m\n\u001b[1;32m    294\u001b[0m     output_path \u001b[39m=\u001b[39m url_or_filename\n",
      "File \u001b[0;32m~/miniconda3/envs/autoeval/lib/python3.9/site-packages/transformers/utils/hub.py:545\u001b[0m, in \u001b[0;36mget_from_cache\u001b[0;34m(url, cache_dir, force_download, proxies, etag_timeout, resume_download, user_agent, use_auth_token, local_files_only)\u001b[0m\n\u001b[1;32m    539\u001b[0m                 \u001b[39mraise\u001b[39;00m \u001b[39mFileNotFoundError\u001b[39;00m(\n\u001b[1;32m    540\u001b[0m                     \u001b[39m\"\u001b[39m\u001b[39mCannot find the requested files in the cached path and outgoing traffic has been\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    541\u001b[0m                     \u001b[39m\"\u001b[39m\u001b[39m disabled. To enable model look-ups and downloads online, set \u001b[39m\u001b[39m'\u001b[39m\u001b[39mlocal_files_only\u001b[39m\u001b[39m'\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    542\u001b[0m                     \u001b[39m\"\u001b[39m\u001b[39m to False.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    543\u001b[0m                 )\n\u001b[1;32m    544\u001b[0m             \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 545\u001b[0m                 \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m    546\u001b[0m                     \u001b[39m\"\u001b[39m\u001b[39mConnection error, and we cannot find the requested files in the cached path.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    547\u001b[0m                     \u001b[39m\"\u001b[39m\u001b[39m Please try again or make sure your Internet connection is on.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    548\u001b[0m                 )\n\u001b[1;32m    550\u001b[0m \u001b[39m# From now on, etag is not None.\u001b[39;00m\n\u001b[1;32m    551\u001b[0m \u001b[39mif\u001b[39;00m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mexists(cache_path) \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m force_download:\n",
      "\u001b[0;31mValueError\u001b[0m: Connection error, and we cannot find the requested files in the cached path. Please try again or make sure your Internet connection is on."
     ]
    }
   ],
   "source": [
    "run_inference(dataset=dataset,model=model,split='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kmeans(df, num_clusters=3):\n",
    "    X = np.array(df['embedding'].tolist())\n",
    "    kclusterer = KMeansClusterer(\n",
    "        num_clusters, distance=nltk.cluster.util.cosine_distance,\n",
    "        repeats=25,avoid_empty_clusters=True)\n",
    "    assigned_clusters = kclusterer.cluster(X, assign_clusters=True)\n",
    "    df['cluster'] = pd.Series(assigned_clusters, index=df.index).astype('int')\n",
    "    df['centroid'] = df['cluster'].apply(lambda x: kclusterer.means()[x])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cluster_errors(data_hl):\n",
    "    merged = pd.DataFrame()\n",
    "    ind=0\n",
    "    num_clusters=0 #cluster count so far\n",
    "    for df in data_hl:\n",
    "        if 'cluster' in df.columns:\n",
    "            df = df.drop(columns=['cluster','centroid'])\n",
    "        kmeans_df = kmeans(df,num_clusters=int(math.sqrt(len(df)/2)))\n",
    "        #print(kmeans_df.loc[kmeans_df['cluster'].idxmax()])\n",
    "        df['cluster'] = kmeans_df['cluster'] + num_clusters\n",
    "        num_clusters=num_clusters + int(math.sqrt(len(df)/2))\n",
    "        ind = ind+1\n",
    "        merged = pd.concat([merged, df], ignore_index=True)\n",
    "    return merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "def generate_groups(merged):\n",
    "    clusters = merged.groupby('cluster')\n",
    "    groups = {x: clusters.get_group(x) for x in clusters.groups}\n",
    "    cluster_acc = {x: accuracy_score(clusters.get_group(x)['label'].values.tolist(), clusters.get_group(x)['pred'].values.tolist()) for x in clusters.groups}\n",
    "    return groups, cluster_acc\n",
    "\n",
    "def dict_zip(*dicts):\n",
    "    all_keys = {k for d in dicts for k in d.keys()}\n",
    "    return {k: [d[k] for d in dicts if k in d] for k in all_keys}\n",
    "\n",
    "def semantic_labeling(groups):\n",
    "    group_labels= {gidx: generate_group_label(cluster) for gidx, cluster in tqdm(groups.items())}\n",
    "    error_groups= dict_zip(group_labels,{k: [len(v), v.iloc[0].label] for k, v in groups.items()})\n",
    "    return error_groups\n",
    "\n",
    "def generate_group_label(cluster):\n",
    "    #instruction = \"In this task, we'll assign a short and precise label to a cluster of documents based on the topics or concepts most relevant to these documents. The documents are all subsets of a sentiment classification dataset. Here are some examples of high-quality labels:\\nDocuments:\\n - Like Clay P who posted before me, I too love pancakes. Though I love chocolate chip pancakes. But like Clay I did not love the ones that I got a the Original Pancake House. Typically, restaurants just don't do it the way I like them and I have come to expect that. Grading OPH on those terms, they did a respectable job. It is definitely a worthwhile destination for a pancake lover.\\n - Very small portions, but good food. Had a perfectly cooked fillet minion, but $60 and they could have served with a toothpick, it was so small.  \\nSame for rest of our party. Excellent salmon, but maybe 5 bites again $60. \\nEverything is a la carte so sides extra. Thai papaya salad good, and key lime pie great.\\n - $65 per person to share ONE ribeye?  Yes you read that correctly.  We had our annual guys trip to vegas and we always try to visit a different steak house every time we visit.  This year was Carnevino.  We were surprised in order to order a ribeye or porterhouse, TWO people had to order it.  We found it a bit odd but we had no problems with it.   We found it even more odd the waiter suggested we have the steaks family style (they cut up the meat and put it on a plate for people to share).   Anyways, a bunch us order different cuts of meat.  NY Strip, Filet Mignon and the folks that wanted port.\\n - This place was a flop. Was visiting a friend in Chandler and found this place on mobile Yelp. Had 68-something reviews and 4/5 stars which almost always means it will be a good place. Boy I was wrong this time.\\n\\nI hate those lengthy reviews talking about unimportant things so I'll stick to the main point. The restaurant itself was clean and you can sit down comfortably.\\n\\nThe waiter highly recommended the pork avodovo plate, which is pork that is served doused with either a green or red sauce with beans and rice on the side. I cannot take spicy food, and the waiter insisted the red version\\n Label: overpriced and spicy food\"\n",
    "    instruction = \"In this task, we'll assign a short and precise label to a cluster of documents based on the topics or concepts most relevant to these documents. The documents are all subsets of a sentiment classification dataset.\\n\"\n",
    "    prompt = build_prompt(instruction, cluster)\n",
    "    resp = openai.Completion.create(\n",
    "        prompt=prompt,\n",
    "        engine='text-davinci-002',\n",
    "        #frequency_penalty = 1.2,\n",
    "    )\n",
    "    label = resp['choices'][0]['text']\n",
    "    return label.strip()\n",
    "\n",
    "#code to build prompt and query gpt3 for labeling\n",
    "def build_prompt(instruction, cluster_df):\n",
    "    if len(cluster_df)>10:\n",
    "        content = cluster_df['content'].str[:600].tolist()\n",
    "    else:\n",
    "        content = cluster_df['content'].str[:1000].tolist()\n",
    "    examples = '\\n - '.join(content)\n",
    "    text = instruction + '- ' + examples+ '\\n Cluster label:'\n",
    "    return text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df_imdb = pd.read_parquet('./imdb_test_distillbert.parquet')\n",
    "data_df_imdb = data_df_imdb.drop_duplicates(subset=['content'])\n",
    "data_df_imdb['loss'] = data_df_imdb['loss'].astype(float)\n",
    "losses = data_df_imdb['loss']\n",
    "high_loss = losses.quantile(0.99)\n",
    "data_df_imdb['slice'] = 'high-loss'\n",
    "data_df_imdb['slice'] = data_df_imdb['slice'].where(data_df_imdb['loss'] > high_loss, 'low-loss')\n",
    "data_hl_imdb = data_df_imdb.drop(data_df_imdb[data_df_imdb['slice'] == 'low-loss'].index)\n",
    "data_ll_imdb = data_df_imdb.drop(data_df_imdb[data_df_imdb['slice'] == 'high-loss'].index) \n",
    "df_list_imdb = [d for _, d in data_hl_imdb.groupby(['label'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = data_df_imdb.drop_duplicates(subset=['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df_imdb = tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8622382730076287"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(data_df_imdb['label'],data_df_imdb['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_lvl0=cluster_errors(df_list_imdb)\n",
    "groups_lvl0,acc_lvl0 = generate_groups(merged_lvl0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_lvl1=cluster_errors(groups_lvl0.values())\n",
    "groups_lvl1,acc_lvl1 = generate_groups(merged_lvl1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\n",
      "33\n"
     ]
    }
   ],
   "source": [
    "print(merged_lvl1['cluster'].max())\n",
    "print(len(groups_lvl1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 33/33 [00:41<00:00,  1.27s/it]\n"
     ]
    }
   ],
   "source": [
    "cluster_labels= semantic_labeling(groups_lvl1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43 0.6976744186046512\n"
     ]
    }
   ],
   "source": [
    "match = data_df_imdb[data_df_imdb['content'].str.contains('hamlet' , case=False)]\n",
    "print(len(match), accuracy_score(match['label'], match['pred']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6        5.065617\n",
       "61       3.967897\n",
       "143      4.062117\n",
       "285      4.036336\n",
       "313      4.622490\n",
       "           ...   \n",
       "24580    3.634398\n",
       "24605    4.533525\n",
       "24659    3.830003\n",
       "24708    4.107304\n",
       "24751    4.207838\n",
       "Name: loss, Length: 493, dtype: float64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses.loc[losses>high_loss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses = data_df_imdb['loss']\n",
    "high_loss = losses.quantile(0.98)\n",
    "loss_weights = np.where(losses > high_loss,losses,0.0)\n",
    "loss_weights = loss_weights / loss_weights.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24644"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_df_imdb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24644/24644 [00:16<00:00, 1519.28it/s]\n"
     ]
    }
   ],
   "source": [
    "unique_tokens = []\n",
    "tokens = []\n",
    "for row in tqdm(data_df_imdb['content']):\n",
    "    tokenized = tokenizer(row,padding=True, return_tensors='pt', truncation=True)\n",
    "    tokens.append(tokenized['input_ids'].flatten())\n",
    "    unique_tokens.append(torch.unique(tokenized['input_ids']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        9.110757e-06\n",
       "1        3.830380e-04\n",
       "2        1.652545e-06\n",
       "3        7.110300e-05\n",
       "4        1.504268e-04\n",
       "             ...     \n",
       "24816    1.350308e-05\n",
       "24828    2.056361e-06\n",
       "24829    6.765310e-07\n",
       "24830    6.537001e-07\n",
       "24831    6.824863e-06\n",
       "Name: loss, Length: 24644, dtype: float64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24644"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_df_imdb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def frequent_tokens(data, tokenizer, loss_quantile=0.98, top_k=200, smoothing=0.005):\n",
    "    unique_tokens = []\n",
    "    tokens = []\n",
    "    for row in tqdm(data['content']):\n",
    "        tokenized = tokenizer(row, padding=True, truncation=True, return_tensors='pt')\n",
    "        tokens.append(tokenized['input_ids'].flatten())\n",
    "        unique_tokens.append(torch.unique(tokenized['input_ids']))\n",
    "    losses = data['loss'].astype(float)\n",
    "    high_loss = losses.quantile(loss_quantile)\n",
    "    loss_weights = np.where(losses > high_loss,losses,0.0)\n",
    "    loss_weights = loss_weights / loss_weights.sum()\n",
    "\n",
    "    token_frequencies = defaultdict(float)\n",
    "    token_frequencies_error = defaultdict(float)\n",
    "    weights_uniform = np.full_like(loss_weights, 1 / len(loss_weights))\n",
    "\n",
    "    for i in tqdm(range(len(data))):\n",
    "        for token in unique_tokens[i]:\n",
    "            token_frequencies[token.item()] += weights_uniform[i]\n",
    "            token_frequencies_error[token.item()] += loss_weights[i]\n",
    "\n",
    "    token_lrs = {k: (smoothing+token_frequencies_error[k]) / (\n",
    "        smoothing+token_frequencies[k]) for k in token_frequencies}\n",
    "    tokens_sorted = list(map(lambda x: x[0], sorted(\n",
    "        token_lrs.items(), key=lambda x: x[1])[::-1]))\n",
    "\n",
    "    top_tokens = []\n",
    "    for i, (token) in enumerate(tokens_sorted[:top_k]):\n",
    "        top_tokens.append(['%10s' % (tokenizer.decode(token)), '%.4f' % (token_frequencies[token]), '%.4f' % (\n",
    "            token_frequencies_error[token]), '%4.2f' % (token_lrs[token])])\n",
    "    return pd.DataFrame(top_tokens, columns=['token', 'freq', 'error-freq', 'ratio'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24644/24644 [00:15<00:00, 1551.93it/s]\n",
      "100%|██████████| 24644/24644 [00:05<00:00, 4428.85it/s]\n"
     ]
    }
   ],
   "source": [
    "commontokens = frequent_tokens(data_df_imdb,AutoTokenizer.from_pretrained('lvwerra/distilbert-imdb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "commontokens.to_parquet('./assets/data/imdb_test_distillbert_tokens.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('autoeval')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f312122f3ed45eefb96eb2533558f359e32fabfee58d97447889ae824b25bd02"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}