Add application file

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Koushik Roy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,59 @@
----
-title: Portrait Photo Generator
-emoji: 📚
-colorFrom: blue
-colorTo: purple
-sdk: gradio
-sdk_version: 3.1.1
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Portrait Photo Generator
+
+In this repo we are going to use image segmentation model(facebook/detr-resnet-50-panoptic) to generate portrait photo of any object present in a photo. Then we are going to make a deployable app using gradio and finally, we will host it in hugging face hub spaces.
+
+## User Interface
+Here is a screenshot of the interface. The app has been hosted in hugging face spaces. [Click Here]() to explore the deployed version.
+![Interface Snapshot](interface_snapshot.png)
+> You can drag and drop images in the top left image box. After you drop an image the image segmentaion model will process the image and provide a list of objects in the drowpdown menu in the right. Now, you can choose any object from the list and that objet will be foucsed in the Output image box. You can adjust the slider to adject the strength of the background blur. The whole interface is reactive and anything you change will trigger an auto refresh. You may find the example images in lower right useful.
+
+## Protrait Image Generation Steps:
+- Load segmentation model, pass your image, generate the masks and labels for each object present in the image.
+- Make a list of objects from the "label" keys in the prediction dictionaries.
+- Add a number identifier so that each object name can be unique. This is important for selecting a object.
+- Image segmentation and background blurring method for selected object in the image:
+    - Take the mask provided by the segmentation model in the earlier stage.
+    - Divide by 255 to make the range 0 to 1.
+    - At this point the mask is 1 channel. Make it three channel by coping the single channel three times.
+    - Element wise multiply the input image and the three channel mask. 
+    This will give an image where only the segmented part of the image will be present. This means, only the selected object pixels will be intact and other pixels will be black (0).
+    - Now, take the original image and blur it using any kind of blurring kernel. Here I used Gaussian Blur.
+    - After that, create a invert of the three channel mask created the previous steps.
+    - Element wise multiply the invert mask and blurred image. 
+    This will give an image where the blurred background of the selected object will be present and 
+    the pixels associate with the object of interest will be blank/black/value=0.
+    - Then, add up the segmented image and reverse segmented (blurred background portion). 
+    This will give the desired portrait photo looking output.
+    - Smoothen the image.
+
+## Gradio GUI Steps:
+- Used block element for more control over the interface.
+- Used image.change, slider.change and dropdown.change event handler for generating output each time any of these changes.
+
+## Requirements
+```
+gradio
+numpy
+Pillow
+torch
+timm
+transformers
+```
+```
+pip install -r requirements.txt
+```
+
+## Run App
+```
+python app.py
+```
+
+## Files
+| File | Contents |
+|---|--- |
+| [Segmentation Notebook](segmentation.ipynb) | Here the segmentation model to background blur part has been shown step by step.|
+| [Gradio Notebook](gradio.ipynb) | You may experiment with different componets of a gradio app.|
+| [Gradio App](app.py) | It is the main file of the deploed version of the portrait photo generator app.|
+
+
+

app.py

@@ -0,0 +1,119 @@
+# Portrait Photo Generator App
+
+# Imports
+from PIL import Image, ImageFilter
+import numpy as np
+from transformers import pipeline
+import gradio as gr
+import os
+
+model = pipeline("image-segmentation", model="facebook/detr-resnet-50-panoptic")
+
+pred = []
+
+def img_resize(image):
+    width = 1280
+    width_percent = (width / float(image.size[0]))
+    height = int((float(image.size[1]) * float(width_percent)))
+    return image.resize((width, height))
+
+def image_objects(image):
+    global pred
+    image = img_resize(image)
+    pred = model(image)
+    pred_object_list = [str(i)+'_'+x['label'] for i, x in enumerate(pred)]
+    return gr.Dropdown.update(choices = pred_object_list, interactive = True)
+
+def blurr_object(image, object, blur_strength):
+    image = img_resize(image)
+
+    object_number = int(object.split('_')[0])
+    mask_array = np.asarray(pred[object_number]['mask'])/255
+    image_array = np.asarray(image)
+
+    mask_array_three_channel = np.zeros_like(image_array)
+    mask_array_three_channel[:,:,0] = mask_array
+    mask_array_three_channel[:,:,1] = mask_array
+    mask_array_three_channel[:,:,2] = mask_array
+
+    segmented_image = image_array*mask_array_three_channel
+
+    blur_image = np.asarray(image.filter(ImageFilter.GaussianBlur(radius=blur_strength)))
+    mask_array_three_channel_invert = 1-mask_array_three_channel
+    blur_image_reverse_mask = blur_image*mask_array_three_channel_invert
+
+    blurred_output_image = Image.fromarray((blur_image_reverse_mask).astype(np.uint8)+segmented_image.astype(np.uint8))
+    for _ in range(int(blur_strength//2.5)):
+        blurred_output_image = blurred_output_image.filter(ImageFilter.SMOOTH_MORE)
+    return blurred_output_image
+
+app = gr.Blocks()
+
+
+with app:
+    gr.Markdown(
+            """
+            ## Portrait Photo Generator
+            - Create stunning portrait photos by blurring the background of your selected object.
+            - Adjust the blurring strength using the slider.
+            """)
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(
+            """
+            ### Input Image
+            """)
+            image_input = gr.Image(type="pil")
+            
+
+        with gr.Column():
+            with gr.Row():
+                gr.Markdown(
+                """
+                ### Found Objects
+                """)
+            with gr.Row():
+                blur_slider = gr.Slider(minimum=0.5, maximum=10, value=3, label="Adject Blur Strength")
+            with gr.Row():
+                object_output = gr.Dropdown(label="Select Object From Dropdown")
+
+    
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(
+                """
+                ### Blurred Image Output
+                """)
+            image_output = gr.Image()
+        with gr.Column():
+            gr.Markdown(
+                """
+                ### Example Images
+                """)
+            gr.Examples(
+                examples=[
+                "test_images/dog_horse_cowboy.jpg",
+                "test_images/woman_and_dog.jpg",
+                "test_images/family_in_sofa.jpg",
+                "test_images/group_of_friends.jpg",
+                "test_images/people_group.jpg"
+                ],
+                fn=image_objects,
+                inputs=image_input, 
+                outputs=object_output)
+
+    image_input.change(fn=image_objects, 
+        inputs=image_input, 
+        outputs=object_output
+        )
+
+    object_output.change(fn=blurr_object, 
+        inputs=[image_input, object_output, blur_slider],
+        outputs=image_output)
+
+    blur_slider.change(fn=blurr_object, 
+        inputs=[image_input, object_output, blur_slider],
+        outputs=image_output)
+    
+
+app.launch()

gradio.ipynb

@@ -0,0 +1,209 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Imports\n",
+    "from PIL import Image, ImageFilter\n",
+    "import numpy as np\n",
+    "from transformers import pipeline\n",
+    "import gradio as gr\n",
+    "import os\n",
+    "\n",
+    "model = pipeline(\"image-segmentation\", model=\"facebook/detr-resnet-50-panoptic\")\n",
+    "\n",
+    "pred = []\n",
+    "\n",
+    "def img_resize(image):\n",
+    "    width = 1000\n",
+    "    width_percent = (width / float(image.size[0]))\n",
+    "    height = int((float(image.size[1]) * float(width_percent)))\n",
+    "    return image.resize((width, height))\n",
+    "\n",
+    "def image_objects(image):\n",
+    "    global pred\n",
+    "    image = img_resize(image)\n",
+    "    pred = model(image)\n",
+    "    pred_object_list = [str(i)+'_'+x['label'] for i, x in enumerate(pred)]\n",
+    "    return gr.Dropdown.update(choices = pred_object_list, interactive = True)\n",
+    "\n",
+    "def blurr_object(image, object, blur_strength):\n",
+    "    image = img_resize(image)\n",
+    "\n",
+    "    object_number = int(object.split('_')[0])\n",
+    "    mask_array = np.asarray(pred[object_number]['mask'])/255\n",
+    "    image_array = np.asarray(image)\n",
+    "\n",
+    "    mask_array_three_channel = np.zeros_like(image_array)\n",
+    "    mask_array_three_channel[:,:,0] = mask_array\n",
+    "    mask_array_three_channel[:,:,1] = mask_array\n",
+    "    mask_array_three_channel[:,:,2] = mask_array\n",
+    "\n",
+    "    segmented_image = image_array*mask_array_three_channel\n",
+    "\n",
+    "    blur_image = np.asarray(image.filter(ImageFilter.GaussianBlur(radius=blur_strength)))\n",
+    "    mask_array_three_channel_invert = 1-mask_array_three_channel\n",
+    "    blur_image_reverse_mask = blur_image*mask_array_three_channel_invert\n",
+    "\n",
+    "    blurred_output_image = Image.fromarray((blur_image_reverse_mask).astype(np.uint8)+segmented_image.astype(np.uint8))\n",
+    "\n",
+    "    return blurred_output_image\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7862/\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7862/\" width=\"900\" height=\"500\" allow=\"autoplay; camera; microphone;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Traceback (most recent call last):\n",
+      "  File \"/home/nybsysml/anaconda3/envs/mediapipe3_7/lib/python3.7/site-packages/gradio/routes.py\", line 256, in run_predict\n",
+      "    fn_index, raw_input, username, session_state\n",
+      "  File \"/home/nybsysml/anaconda3/envs/mediapipe3_7/lib/python3.7/site-packages/gradio/blocks.py\", line 599, in process_api\n",
+      "    predictions, duration = await self.call_function(fn_index, processed_input)\n",
+      "  File \"/home/nybsysml/anaconda3/envs/mediapipe3_7/lib/python3.7/site-packages/gradio/blocks.py\", line 515, in call_function\n",
+      "    block_fn.fn, *processed_input, limiter=self.limiter\n",
+      "  File \"/home/nybsysml/anaconda3/envs/mediapipe3_7/lib/python3.7/site-packages/anyio/to_thread.py\", line 32, in run_sync\n",
+      "    func, *args, cancellable=cancellable, limiter=limiter\n",
+      "  File \"/home/nybsysml/anaconda3/envs/mediapipe3_7/lib/python3.7/site-packages/anyio/_backends/_asyncio.py\", line 937, in run_sync_in_worker_thread\n",
+      "    return await future\n",
+      "  File \"/home/nybsysml/anaconda3/envs/mediapipe3_7/lib/python3.7/site-packages/anyio/_backends/_asyncio.py\", line 867, in run\n",
+      "    result = context.run(func, *args)\n",
+      "  File \"/tmp/ipykernel_6335/646230931.py\", line 20, in image_objects\n",
+      "    image = img_resize(image)\n",
+      "  File \"/tmp/ipykernel_6335/646230931.py\", line 14, in img_resize\n",
+      "    width_percent = (width / float(image.size[0]))\n",
+      "AttributeError: 'NoneType' object has no attribute 'size'\n"
+     ]
+    }
+   ],
+   "source": [
+    "app = gr.Blocks()\n",
+    "\n",
+    "\n",
+    "with app:\n",
+    "    gr.Markdown(\n",
+    "            \"\"\"\n",
+    "            ## Portrait Photo Generator\n",
+    "            - Create stunning portrait photos by blurring the background of your selected object.\n",
+    "            - Adjust the blurring strength using the slider.\n",
+    "            \"\"\")\n",
+    "    with gr.Row():\n",
+    "        with gr.Column():\n",
+    "            gr.Markdown(\n",
+    "            \"\"\"\n",
+    "            ### Input Image\n",
+    "            \"\"\")\n",
+    "            image_input = gr.Image(type=\"pil\")\n",
+    "            \n",
+    "\n",
+    "        with gr.Column():\n",
+    "            with gr.Row():\n",
+    "                gr.Markdown(\n",
+    "                \"\"\"\n",
+    "                ### Found Objects\n",
+    "                \"\"\")\n",
+    "            with gr.Row():\n",
+    "                blur_slider = gr.Slider(minimum=0.5, maximum=10, value=3, label=\"Adject Blur Strength\")\n",
+    "            with gr.Row():\n",
+    "                object_output = gr.Dropdown(label=\"Select Object From Dropdown\")\n",
+    "\n",
+    "    \n",
+    "    with gr.Row():\n",
+    "        with gr.Column():\n",
+    "            gr.Markdown(\n",
+    "                \"\"\"\n",
+    "                ### Blurred Image Output\n",
+    "                \"\"\")\n",
+    "            image_output = gr.Image()\n",
+    "        with gr.Column():\n",
+    "            gr.Markdown(\n",
+    "                \"\"\"\n",
+    "                ### Example Images\n",
+    "                \"\"\")\n",
+    "            gr.Examples(\n",
+    "                examples=[\n",
+    "                \"test_images/dog_horse_cowboy.jpg\",\n",
+    "                \"test_images/woman_and_dog.jpg\",\n",
+    "                \"test_images/family_in_sofa.jpg\",\n",
+    "                \"test_images/group_of_friends.jpg\",\n",
+    "                \"test_images/people_group.jpg\"\n",
+    "                ],\n",
+    "                fn=image_objects,\n",
+    "                inputs=image_input, \n",
+    "                outputs=object_output)\n",
+    "\n",
+    "    image_input.change(fn=image_objects, \n",
+    "        inputs=image_input, \n",
+    "        outputs=object_output\n",
+    "        )\n",
+    "\n",
+    "    object_output.change(fn=blurr_object, \n",
+    "        inputs=[image_input, object_output, blur_slider],\n",
+    "        outputs=image_output)\n",
+    "\n",
+    "    blur_slider.change(fn=blurr_object, \n",
+    "        inputs=[image_input, object_output, blur_slider],\n",
+    "        outputs=image_output)\n",
+    "    \n",
+    "\n",
+    "app.launch(debug=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.7 ('base')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "d03b47a5a9bfc09262d9700ac41deff9ead9ff3e21241983433bdf2900140cd2"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,6 @@
+gradio
+numpy
+Pillow
+torch
+timm
+transformers