Upload sd_token_similarity_calculator.ipynb

sd_token_similarity_calculator.ipynb

@@ -28,7 +28,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# @title Load/initialize values\n",
+        "# @title ✳️ Load/initialize values\n",
         "# Load the tokens into the colab\n",
         "!git clone https://huggingface.co/datasets/codeShare/sd_tokens\n",
         "import torch\n",
@@ -116,23 +116,10 @@
       "metadata": {
         "id": "Ch9puvwKH1s3",
         "collapsed": true,
-        "cellView": "form",
-        "outputId": "9a9d4274-a633-464b-e1fb-06a33f3dd873",
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        }
+        "cellView": "form"
       },
-      "execution_count": 59,
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "fatal: destination path 'sd_tokens' already exists and is not an empty directory.\n",
-            "/content/sd_tokens\n"
-          ]
-        }
-      ]
+      "execution_count": null,
+      "outputs": []
     },
     {
       "cell_type": "code",
@@ -278,7 +265,8 @@
         "#Print the sorted list from above result"
       ],
       "metadata": {
-        "id": "iWeFnT1gAx6A"
+        "id": "iWeFnT1gAx6A",
+        "cellView": "form"
       },
       "execution_count": null,
       "outputs": []
@@ -315,7 +303,8 @@
       ],
       "metadata": {
         "id": "QQOjh5BvnG8M",
-        "collapsed": true
+        "collapsed": true,
+        "cellView": "form"
       },
       "execution_count": null,
       "outputs": []
@@ -323,14 +312,497 @@
     {
       "cell_type": "code",
       "source": [
-        "# @title 🪐 Find similiar prompt\n",
-        "# @markdown Prompt A to match against\n",
+        "# @title 🪐🖼️ -> 📝 Image to prompt : Add single token to existing prompt to match image\n",
+        "from google.colab import files\n",
+        "def getLocalFiles():\n",
+        "    _files = files.upload()\n",
+        "    if len(_files) >0:\n",
+        "       for k,v in _files.items():\n",
+        "         open(k,'wb').write(v)\n",
+        "\n",
+        "#Get image\n",
+        "# You can use \"http://images.cocodataset.org/val2017/000000039769.jpg\" for testing\n",
+        "url = \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {\"type\":\"string\",\"placeholder\":\"leave empty for local upload\"}\n",
+        "from PIL import Image\n",
+        "import requests\n",
+        "if url == \"\":\n",
+        "  image_A = getLocalFiles()\n",
+        "else:\n",
+        "  image_A = Image.open(requests.get(url, stream=True).raw)\n",
+        "\n",
+        "\n",
+        "# Get image features\n",
+        "from transformers import  CLIPProcessor, CLIPModel\n",
+        "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-large-patch14\" , clean_up_tokenization_spaces = True)\n",
+        "model = CLIPModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
+        "inputs = processor(images=image_A, return_tensors=\"pt\")\n",
+        "image_features = model.get_image_features(**inputs)\n",
+        "text_encoding_A = image_features\n",
+        "A =  text_encoding_A[0]\n",
+        "_A = LA.vector_norm(A, ord=2)\n",
+        "prompt_A = \"the image\"\n",
+        "name_A = prompt_A\n",
+        "#-----#\n",
+        "\n",
+        "# @markdown Set conditions for the output\n",
+        "must_start_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
+        "must_contain = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
+        "must_end_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
+        "token_B = must_contain\n",
+        "\n",
+        "# @markdown Limit the search\n",
+        "use_token_padding = True # @param {type:\"boolean\"}\n",
+        "start_search_at_ID = 12500 # @param {type:\"slider\", min:0, max: 49407, step:100}\n",
+        "search_range = 500 # @param {type:\"slider\", min:0, max: 2000, step:100}\n",
+        "restrictions = 'Suffix only' # @param [\"None\", \"Suffix only\", \"Prefix only\"]\n",
+        "\n",
+        "# @markdown Limit char size of included token\n",
+        "min_char_size = 3 # @param {type:\"slider\", min:0, max: 50, step:1}\n",
+        "char_range = 5 # @param {type:\"slider\", min:0, max: 50, step:1}\n",
+        "\n",
+        "#Tokenize input B\n",
+        "from transformers import AutoTokenizer\n",
+        "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
+        "tokenizer_output = tokenizer(text = token_B)\n",
+        "input_ids = tokenizer_output['input_ids']\n",
+        "#-----#\n",
+        "name_B = must_contain\n",
+        "#-----#\n",
+        "\n",
+        "START = start_search_at_ID\n",
+        "RANGE =  min(search_range , 49407 - start_search_at_ID)\n",
+        "\n",
+        "dots = torch.zeros(RANGE)\n",
+        "is_BC = torch.zeros(RANGE)\n",
+        "for index in range(RANGE):\n",
+        "  id_C = START + index\n",
+        "  C = token[id_C]\n",
+        "  _C = LA.vector_norm(C, ord=2)\n",
+        "  name_C = vocab[id_C]\n",
+        "\n",
+        "  # Decide if we should process prefix/suffix tokens\n",
+        "  if name_C.find('</w>')<=-1:\n",
+        "    if restrictions != \"Prefix only\":\n",
+        "      continue\n",
+        "  else:\n",
+        "    if restrictions == \"Prefix only\":\n",
+        "      continue\n",
+        "  #-----#\n",
+        "\n",
+        "  # Decide if char-size is within range\n",
+        "  if len(name_C) < min_char_size:\n",
+        "    continue\n",
+        "  if len(name_C) > min_char_size + char_range:\n",
+        "    continue\n",
+        "  #-----#\n",
+        "\n",
+        "  name_CB = must_start_with + name_C + name_B + must_end_with\n",
+        "  if restrictions == \"Prefix only\":\n",
+        "    name_CB = must_start_with +  name_C + '-' + name_B + must_end_with\n",
+        "  #-----#\n",
+        "  ids_CB = processor.tokenizer(text=name_CB, padding=use_token_padding, return_tensors=\"pt\")\n",
+        "  text_encoding_CB = model.get_text_features(**ids_CB)\n",
+        "  CB = text_encoding_CB[0]\n",
+        "  _CB =  LA.vector_norm(CB, ord=2)\n",
+        "  sim_CB = torch.dot(A,CB)/(_A*_CB)\n",
+        "  #-----#\n",
+        "  if restrictions == \"Prefix only\":\n",
+        "    result = sim_CB\n",
+        "    result = result.item()\n",
+        "    dots[index] = result\n",
+        "    continue\n",
+        "  #-----#\n",
+        "  name_BC = must_start_with + name_B + name_C + must_end_with\n",
+        "  ids_BC = processor.tokenizer(text=name_BC, padding=use_token_padding, return_tensors=\"pt\")\n",
+        "  text_encoding_BC = model.get_text_features(**ids_BC)\n",
+        "  BC = text_encoding_BC[0]\n",
+        "  _BC =  LA.vector_norm(BC, ord=2)\n",
+        "  sim_BC = torch.dot(A,BC)/(_A*_BC)\n",
+        "  #-----#\n",
+        "\n",
+        "  result = sim_CB\n",
+        "  if(sim_BC > sim_CB):\n",
+        "    is_BC[index] = 1\n",
+        "    result = sim_BC\n",
+        "\n",
+        "  #result = absolute_value(result.item())\n",
+        "  result = result.item()\n",
+        "  dots[index] = result\n",
+        "#----#\n",
+        "\n",
+        "sorted, indices = torch.sort(dots,dim=0 , descending=True)\n",
+        "\n",
+        "# @markdown Print options\n",
+        "list_size = 100 # @param {type:'number'}\n",
+        "print_ID = False # @param {type:\"boolean\"}\n",
+        "print_Similarity = True # @param {type:\"boolean\"}\n",
+        "print_Name = True # @param {type:\"boolean\"}\n",
+        "print_Divider = True # @param {type:\"boolean\"}\n",
+        "\n",
+        "\n",
+        "if (print_Divider):\n",
+        "  print('//---//')\n",
+        "\n",
+        "print('')\n",
+        "print(f'These token pairings within the range ID = {START} to ID = {START + RANGE} most closely match the text_encoding for {prompt_A} : ')\n",
+        "print('')\n",
+        "\n",
+        "for index in range(min(list_size,RANGE)):\n",
+        "  id = START + indices[index].item()\n",
+        "  if (print_Name):\n",
+        "    if(is_BC[index]>0):\n",
+        "      print(must_start_with +  name_B + vocab[id] + must_end_with)\n",
+        "    else:\n",
+        "      if restrictions == \"Prefix only\":\n",
+        "        print(must_start_with +   vocab[id] + '-'  + name_B + must_end_with)\n",
+        "      else:\n",
+        "        print(must_start_with +   vocab[id] + name_B + must_end_with)\n",
+        "  if (print_ID):\n",
+        "    print(f'ID = {id}') # IDs\n",
+        "  if (print_Similarity):\n",
+        "    print(f'similiarity = {round(sorted[index].item()*100,2)} %')\n",
+        "  if (print_Divider):\n",
+        "    print('--------')\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n"
+      ],
+      "metadata": {
+        "collapsed": true,
+        "cellView": "form",
+        "id": "fi0jRruI0-tu",
+        "outputId": "6d7e8c39-a117-4b35-acfe-2a128c65aeb7",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 9,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "//---//\n",
+            "\n",
+            "These token pairings within the range ID = 12500 to ID = 13000 most closely match the text_encoding for the prompt \"the image\" : \n",
+            "\n",
+            "sits</w>yellow\n",
+            "similiarity = 23.02 %\n",
+            "--------\n",
+            "neys</w>yellow\n",
+            "similiarity = 19.74 %\n",
+            "--------\n",
+            "cody</w>yellow\n",
+            "similiarity = 18.61 %\n",
+            "--------\n",
+            "wns</w>yellow\n",
+            "similiarity = 18.43 %\n",
+            "--------\n",
+            "java</w>yellow\n",
+            "similiarity = 18.15 %\n",
+            "--------\n",
+            "jj</w>yellow\n",
+            "similiarity = 18.03 %\n",
+            "--------\n",
+            "eno</w>yellow\n",
+            "similiarity = 17.87 %\n",
+            "--------\n",
+            "cled</w>yellow\n",
+            "similiarity = 17.85 %\n",
+            "--------\n",
+            "nom</w>yellow\n",
+            "similiarity = 17.75 %\n",
+            "--------\n",
+            "dads</w>yellow\n",
+            "similiarity = 17.5 %\n",
+            "--------\n",
+            "mil</w>yellow\n",
+            "similiarity = 17.47 %\n",
+            "--------\n",
+            "whom</w>yellow\n",
+            "similiarity = 17.37 %\n",
+            "--------\n",
+            "itv</w>yellow\n",
+            "similiarity = 17.34 %\n",
+            "--------\n",
+            "vibe</w>yellow\n",
+            "similiarity = 17.2 %\n",
+            "--------\n",
+            "noir</w>yellow\n",
+            "similiarity = 17.14 %\n",
+            "--------\n",
+            "yellowarel</w>\n",
+            "similiarity = 17.1 %\n",
+            "--------\n",
+            "#âĢ¦</w>yellow\n",
+            "similiarity = 17.04 %\n",
+            "--------\n",
+            "maya</w>yellow\n",
+            "similiarity = 17.03 %\n",
+            "--------\n",
+            "yellowbam</w>\n",
+            "similiarity = 17.01 %\n",
+            "--------\n",
+            "erts</w>yellow\n",
+            "similiarity = 17.01 %\n",
+            "--------\n",
+            "xc</w>yellow\n",
+            "similiarity = 16.98 %\n",
+            "--------\n",
+            "mob</w>yellow\n",
+            "similiarity = 16.89 %\n",
+            "--------\n",
+            "dees</w>yellow\n",
+            "similiarity = 16.87 %\n",
+            "--------\n",
+            "icc</w>yellow\n",
+            "similiarity = 16.75 %\n",
+            "--------\n",
+            "aly</w>yellow\n",
+            "similiarity = 16.63 %\n",
+            "--------\n",
+            "lis</w>yellow\n",
+            "similiarity = 16.63 %\n",
+            "--------\n",
+            "yellowturf</w>\n",
+            "similiarity = 16.62 %\n",
+            "--------\n",
+            "yellowbaba</w>\n",
+            "similiarity = 16.58 %\n",
+            "--------\n",
+            ":*</w>yellow\n",
+            "similiarity = 16.42 %\n",
+            "--------\n",
+            "inho</w>yellow\n",
+            "similiarity = 16.39 %\n",
+            "--------\n",
+            "yellowhes</w>\n",
+            "similiarity = 16.37 %\n",
+            "--------\n",
+            "nity</w>yellow\n",
+            "similiarity = 16.3 %\n",
+            "--------\n",
+            "lust</w>yellow\n",
+            "similiarity = 16.3 %\n",
+            "--------\n",
+            "ikh</w>yellow\n",
+            "similiarity = 16.26 %\n",
+            "--------\n",
+            "nyt</w>yellow\n",
+            "similiarity = 16.24 %\n",
+            "--------\n",
+            "(+</w>yellow\n",
+            "similiarity = 16.11 %\n",
+            "--------\n",
+            "foto</w>yellow\n",
+            "similiarity = 16.11 %\n",
+            "--------\n",
+            "stl</w>yellow\n",
+            "similiarity = 16.06 %\n",
+            "--------\n",
+            "mick</w>yellow\n",
+            "similiarity = 16.06 %\n",
+            "--------\n",
+            "...@</w>yellow\n",
+            "similiarity = 16.05 %\n",
+            "--------\n",
+            "ugh</w>yellow\n",
+            "similiarity = 16.05 %\n",
+            "--------\n",
+            "gro</w>yellow\n",
+            "similiarity = 16.01 %\n",
+            "--------\n",
+            "wski</w>yellow\n",
+            "similiarity = 16.01 %\n",
+            "--------\n",
+            "ðŁĴ«</w>yellow\n",
+            "similiarity = 15.74 %\n",
+            "--------\n",
+            "deen</w>yellow\n",
+            "similiarity = 15.73 %\n",
+            "--------\n",
+            "assy</w>yellow\n",
+            "similiarity = 15.72 %\n",
+            "--------\n",
+            "mtv</w>yellow\n",
+            "similiarity = 15.72 %\n",
+            "--------\n",
+            "yellowðŁĺ»</w>\n",
+            "similiarity = 15.72 %\n",
+            "--------\n",
+            "yellowfrm</w>\n",
+            "similiarity = 15.65 %\n",
+            "--------\n",
+            "moss</w>yellow\n",
+            "similiarity = 15.64 %\n",
+            "--------\n",
+            "bart</w>yellow\n",
+            "similiarity = 15.61 %\n",
+            "--------\n",
+            "tw</w>yellow\n",
+            "similiarity = 15.51 %\n",
+            "--------\n",
+            "yellowplug</w>\n",
+            "similiarity = 15.46 %\n",
+            "--------\n",
+            "jen</w>yellow\n",
+            "similiarity = 15.45 %\n",
+            "--------\n",
+            "pst</w>yellow\n",
+            "similiarity = 15.43 %\n",
+            "--------\n",
+            "omfg</w>yellow\n",
+            "similiarity = 15.43 %\n",
+            "--------\n",
+            "dine</w>yellow\n",
+            "similiarity = 15.38 %\n",
+            "--------\n",
+            "vern</w>yellow\n",
+            "similiarity = 15.33 %\n",
+            "--------\n",
+            "reno</w>yellow\n",
+            "similiarity = 15.25 %\n",
+            "--------\n",
+            "yellow´</w>\n",
+            "similiarity = 15.14 %\n",
+            "--------\n",
+            "omic</w>yellow\n",
+            "similiarity = 15.14 %\n",
+            "--------\n",
+            "łï¸ı</w>yellow\n",
+            "similiarity = 15.11 %\n",
+            "--------\n",
+            "yellowgis</w>\n",
+            "similiarity = 15.06 %\n",
+            "--------\n",
+            "aunt</w>yellow\n",
+            "similiarity = 15.0 %\n",
+            "--------\n",
+            "joan</w>yellow\n",
+            "similiarity = 14.96 %\n",
+            "--------\n",
+            "anas</w>yellow\n",
+            "similiarity = 14.92 %\n",
+            "--------\n",
+            "ðŁĴĵ</w>yellow\n",
+            "similiarity = 14.9 %\n",
+            "--------\n",
+            "chad</w>yellow\n",
+            "similiarity = 14.89 %\n",
+            "--------\n",
+            "yellowsake</w>\n",
+            "similiarity = 14.88 %\n",
+            "--------\n",
+            "gues</w>yellow\n",
+            "similiarity = 14.84 %\n",
+            "--------\n",
+            "gian</w>yellow\n",
+            "similiarity = 14.84 %\n",
+            "--------\n",
+            "asi</w>yellow\n",
+            "similiarity = 14.83 %\n",
+            "--------\n",
+            "yellowoven</w>\n",
+            "similiarity = 14.82 %\n",
+            "--------\n",
+            "jury</w>yellow\n",
+            "similiarity = 14.79 %\n",
+            "--------\n",
+            "blvd</w>yellow\n",
+            "similiarity = 14.75 %\n",
+            "--------\n",
+            "omez</w>yellow\n",
+            "similiarity = 14.72 %\n",
+            "--------\n",
+            "yellowyang</w>\n",
+            "similiarity = 14.7 %\n",
+            "--------\n",
+            "gu</w>yellow\n",
+            "similiarity = 14.48 %\n",
+            "--------\n",
+            "yellowova</w>\n",
+            "similiarity = 14.45 %\n",
+            "--------\n",
+            "yellowinez</w>\n",
+            "similiarity = 14.44 %\n",
+            "--------\n",
+            "pei</w>yellow\n",
+            "similiarity = 14.44 %\n",
+            "--------\n",
+            "ãĢIJ</w>yellow\n",
+            "similiarity = 14.43 %\n",
+            "--------\n",
+            "ãĢij</w>yellow\n",
+            "similiarity = 14.43 %\n",
+            "--------\n",
+            "ðŁĮŀ</w>yellow\n",
+            "similiarity = 14.36 %\n",
+            "--------\n",
+            "ðŁĺĿ</w>yellow\n",
+            "similiarity = 14.27 %\n",
+            "--------\n",
+            "troy</w>yellow\n",
+            "similiarity = 14.16 %\n",
+            "--------\n",
+            "pale</w>yellow\n",
+            "similiarity = 14.14 %\n",
+            "--------\n",
+            "boi</w>yellow\n",
+            "similiarity = 14.11 %\n",
+            "--------\n",
+            "nn</w>yellow\n",
+            "similiarity = 14.08 %\n",
+            "--------\n",
+            "âı°</w>yellow\n",
+            "similiarity = 14.01 %\n",
+            "--------\n",
+            "ooth</w>yellow\n",
+            "similiarity = 13.93 %\n",
+            "--------\n",
+            "pied</w>yellow\n",
+            "similiarity = 13.9 %\n",
+            "--------\n",
+            "bola</w>yellow\n",
+            "similiarity = 13.79 %\n",
+            "--------\n",
+            "âŀ¡</w>yellow\n",
+            "similiarity = 13.77 %\n",
+            "--------\n",
+            "rena</w>yellow\n",
+            "similiarity = 13.75 %\n",
+            "--------\n",
+            "dley</w>yellow\n",
+            "similiarity = 13.73 %\n",
+            "--------\n",
+            "evan</w>yellow\n",
+            "similiarity = 13.67 %\n",
+            "--------\n",
+            "pony</w>yellow\n",
+            "similiarity = 13.63 %\n",
+            "--------\n",
+            "rene</w>yellow\n",
+            "similiarity = 13.62 %\n",
+            "--------\n",
+            "mock</w>yellow\n",
+            "similiarity = 13.57 %\n",
+            "--------\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title 🪐📝 Prompt to prompt : Add single token to existing prompt to match another prompt\n",
+        "# @markdown Write a text to match against...\n",
         "prompt_A = \"photo of a banana\" # @param {\"type\":\"string\",\"placeholder\":\"Write a prompt\"}\n",
+        "\n",
         "# @markdown Set conditions for the output\n",
-        "must_start_with = \"bendy \" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
+        "must_start_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
         "must_contain = \"yellow\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
-        "must_end_with = \" on a table\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
-        "\n",
+        "must_end_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
         "token_B = must_contain\n",
         "\n",
         "# @markdown Limit the search\n",
@@ -343,7 +815,6 @@
         "min_char_size = 3 # @param {type:\"slider\", min:0, max: 50, step:1}\n",
         "char_range = 5 # @param {type:\"slider\", min:0, max: 50, step:1}\n",
         "\n",
-        "\n",
         "#Tokenize input B\n",
         "from transformers import AutoTokenizer\n",
         "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
@@ -427,8 +898,6 @@
         "  dots[index] = result\n",
         "#----#\n",
         "\n",
-        "\n",
-        "\n",
         "sorted, indices = torch.sort(dots,dim=0 , descending=True)\n",
         "\n",
         "# @markdown Print options\n",
@@ -464,6 +933,7 @@
         "    print('--------')"
       ],
       "metadata": {
+        "cellView": "form",
         "id": "uDtcm-l8UCJk"
       },
       "execution_count": null,
@@ -901,7 +1371,9 @@
         "\n",
         "There might be some updates in the future with features not mentioned here.\n",
         "\n",
-        "//---//"
+        "//---//\n",
+        "\n",
+        "https://codeandlife.com/2023/01/26/mastering-the-huggingface-clip-model-how-to-extract-embeddings-and-calculate-similarity-for-text-and-images/"
       ],
       "metadata": {
         "id": "njeJx_nSSA8H"