Upload app.py

app.py

@@ -0,0 +1,255 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Image captioning with ViT+GPT2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "f:\\Image caption genrerator\\image_caption\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from PIL import Image\n",
+    "from transformers import VisionEncoderDecoderModel, ViTFeatureExtractor, PreTrainedTokenizerFast\n",
+    "import requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = VisionEncoderDecoderModel.from_pretrained(\"nlpconnect/vit-gpt2-image-captioning\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "f:\\Image caption genrerator\\image_caption\\lib\\site-packages\\transformers\\models\\vit\\feature_extraction_vit.py:28: FutureWarning: The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use ViTImageProcessor instead.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "vit_feature_extactor = ViTFeatureExtractor.from_pretrained(\"google/vit-base-patch16-224-in21k\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
+      "The tokenizer class you load from this checkpoint is 'GPT2Tokenizer'. \n",
+      "The class this function is called from is 'PreTrainedTokenizerFast'.\n"
+     ]
+    }
+   ],
+   "source": [
+    "tokenizer = PreTrainedTokenizerFast.from_pretrained(\"distilgpt2\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#url = 'https://d2gp644kobdlm6.cloudfront.net/wp-content/uploads/2016/06/bigstock-Shocked-and-surprised-boy-on-t-113798588-300x212.jpg'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# with Image.open(requests.get(url, stream=True).raw) as img:\n",
+    "#     pixel_values = vit_feature_extactor(images=img, return_tensors=\"pt\").pixel_values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "f:\\Image caption genrerator\\image_caption\\lib\\site-packages\\transformers\\generation\\utils.py:1346: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "# encoder_outputs = model.generate(pixel_values.to('cpu'),num_beams = 5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# generated_senetences = tokenizer.batch_decode(encoder_outputs, skip_special_tokens=True,)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['a young boy sitting in front of a laptop computer ']"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# generated_senetences"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'a young boy sitting in front of a laptop computer '"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# generated_senetences[0].split(\".\")[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def vit2distilgpt2(img):\n",
+    "    pixel_values = vit_feature_extactor(images=img, return_tensors=\"pt\").pixel_values\n",
+    "    encoder_outputs = generated_ids = model.generate(pixel_values.to('cpu'),num_beams=5)\n",
+    "    generated_senetences = tokenizer.batch_decode(encoder_outputs, skip_special_tokens=True)\n",
+    "\n",
+    "    return(generated_senetences[0].split('.')[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gradio as gr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'gr' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[2], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m inputs \u001b[39m=\u001b[39m [\n\u001b[1;32m----> 2\u001b[0m     gr\u001b[39m.\u001b[39minputs\u001b[39m.\u001b[39mImage(\u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mpil\u001b[39m\u001b[39m\"\u001b[39m,label\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mOriginal Images\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m      3\u001b[0m ]\n\u001b[0;32m      5\u001b[0m outputs \u001b[39m=\u001b[39m [\n\u001b[0;32m      6\u001b[0m     gr\u001b[39m.\u001b[39moutputs\u001b[39m.\u001b[39mTextbox(label \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mCaption\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m      7\u001b[0m ]\n\u001b[0;32m      9\u001b[0m title \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mImage Captioning using ViT + GPT2\u001b[39m\u001b[39m\"\u001b[39m\n",
+      "\u001b[1;31mNameError\u001b[0m: name 'gr' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "inputs = [\n",
+    "    gr.inputs.Image(type=\"pil\",label=\"Original Images\")\n",
+    "]\n",
+    "\n",
+    "outputs = [\n",
+    "    gr.outputs.Textbox(label = \"Caption\")\n",
+    "]\n",
+    "\n",
+    "title = \"Image Captioning using ViT + GPT2\"\n",
+    "description = \"ViT and GPT2 are used to generate Image Caption for the uploaded image.COCO DataSet is used for Training\"\n",
+    "examples = [\n",
+    "    [\".Image1.png\"],\n",
+    "    [\".Image2.png\"],\n",
+    "    [\".Image3.png\"]\n",
+    "]\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "gr.Interface(\n",
+    "    vit2distilgpt2,\n",
+    "    inputs,\n",
+    "    outputs,\n",
+    "    title=title,\n",
+    "    description=description,\n",
+    "    examples=examples,\n",
+    "    theme=\"huggingface\",\n",
+    ").launch(debug=True, enable_queue=True, share=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}