Upload folder using huggingface_hub

requirements.txt

@@ -0,0 +1,5 @@
+gradio==4.44.1
+pymongo==4.8.0
+requests==2.32.3
+python-dotenv==1.0.1
+pillow==10.4.0

.idea/gradio.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/enviro" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (gradio)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/gradio.iml" filepath="$PROJECT_DIR$/.idea/gradio.iml" />
+    </modules>
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="8697917c-9f93-4f94-859f-bab053b785ea" name="Changes" comment="" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
+  <component name="ProjectId" id="2lbyoOwwUV5Z6jiUjzA95ejUu03" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "WebServerToolWindowFactoryState": "false",
+    "last_opened_file_path": "/Users/piadonabauer/PycharmProjects/gradio",
+    "node.js.detected.package.eslint": "true",
+    "node.js.detected.package.tslint": "true",
+    "node.js.selected.package.eslint": "(autodetect)",
+    "node.js.selected.package.tslint": "(autodetect)",
+    "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable",
+    "vue.rearranger.settings.migration": "true"
+  }
+}]]></component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$" />
+    </key>
+    <key name="MoveFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$" />
+      <recent name="$PROJECT_DIR$/data" />
+    </key>
+  </component>
+  <component name="RunManager">
+    <configuration name="main" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
+      <module name="gradio" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/main.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="8697917c-9f93-4f94-859f-bab053b785ea" name="Changes" comment="" />
+      <created>1725463244919</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1725463244919</updated>
+      <workItem from="1725463246928" duration="18213000" />
+      <workItem from="1725897141437" duration="4873000" />
+      <workItem from="1727438790253" duration="7494000" />
+      <workItem from="1727876890592" duration="1494000" />
+      <workItem from="1730908139516" duration="823000" />
+      <workItem from="1730928642179" duration="5000" />
+      <workItem from="1732371650265" duration="677000" />
+      <workItem from="1736170501354" duration="2227000" />
+      <workItem from="1736236372042" duration="13000" />
+      <workItem from="1736236479831" duration="15221000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+</project>

README.md

@@ -1,12 +1,9 @@
 ---
-title: Image Edit Annotation
-emoji: 📉
-colorFrom: gray
-colorTo: gray
+title: Image-Edit-Annotation
+emoji: 🔍
+colorFrom: blue
+colorTo: pink
 sdk: gradio
-sdk_version: 5.10.0
 app_file: app.py
 pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---

app.py

@@ -0,0 +1,193 @@
+import gradio as gr
+import os
+from PIL import Image
+from pymongo import MongoClient
+import requests
+from io import BytesIO
+from dotenv import load_dotenv
+
+instruction_beginning = """
+## 🔍 Evaluation of AI Quality
+
+\n**Background**
+
+\nIn this task, you will evaluate the quality of image edits based on a textual instruction and two input images regarding three aspects: **Instruction-Edit Alignment**, **Visual Quality** and **Consistency**.
+Each aspect should be rated on a scale from 1 to 10, where 1 indicates 'very poor' and 10 represents 'excellent'.
+
+Please ensure you have read the detailed instructions provided in this [document](https://www.canva.com/design/DAGP0UTTygI/rYkYZtLUipuKbXPbRcj9kQ/edit?utm_content=DAGP0UTTygI&utm_campaign=designshare&utm_medium=link2&utm_source=sharebutton) before starting the labeling process.
+
+\n**Labeling**
+
+\nPlease enter a nickname to avoid repeating triplets. Make sure to remember this nickname for future sessions. If you choose not to enter a nickname, your ratings will not be saved.
+"""
+
+alignment_info = """
+How well does the edited area align with the text instruction? (e.g. numbers, colors, and objects)
+"""
+
+quality_info = """
+How realistic and aesthetically pleasing is the edited area? (e.g. color realism and overall aesthetics)
+"""
+
+consistency_info = """
+How seamlessly does the edit integrate with the rest of the original image? (e.g. consistency in style, lighting, logic, and spatial coherence)
+"""
+
+overall_info = """
+How do you perceive and like the edit as a whole, how well does it meet your expectations and complements the original image?
+"""
+
+# load_dotenv()
+
+mongo_user = os.getenv('MONGO_USER')
+mongo_password = os.getenv('MONGO_PASSWORD')
+cluster_url = os.getenv('MONGO_CLUSTER_URL')
+gradio_user = os.getenv('GRADIO_USER')
+gradio_password = os.getenv('GRADIO_PASSWORD')
+
+connection_url = f"mongodb+srv://{mongo_user}:{mongo_password}@{cluster_url}"
+client = MongoClient(connection_url)
+db = client["thesis"]
+collection = db["labeling"]
+
+
+def download_image(url):
+    """Download image from a given URL."""
+    response = requests.get(url)
+    response.raise_for_status()
+    return Image.open(BytesIO(response.content))
+
+
+def fetch_random_entry(annotator):
+    """Fetch a random entry from the database that hasn't been rated three times yet."""
+    pipeline = [
+        {
+            "$addFields": {
+                "ratings_count": {"$size": "$ratings"}
+            }
+        },
+        {
+            "$match": {
+                "ratings_count": {"$lt": 5},
+                "ratings.rater": {"$ne": annotator}  # exclude entries where rater is the specified annotator
+            }
+        },
+        {"$sample": {"size": 1}}  # randomly select one entry
+    ]
+
+    results = list(collection.aggregate(pipeline))
+    return results[0] if results else None
+
+
+def save_rating(entry_id, turn, annotator, alignment, quality, consistency, overall):
+    """Save the given ratings into the database."""
+    if annotator:
+        rating = {
+            "rater": annotator,
+            "alignment": alignment,
+            "quality": quality,
+            "consistency": consistency,
+            "overall": overall
+        }
+
+        collection.update_one(
+            {"meta_information.id": int(entry_id), "meta_information.turn": int(turn)},
+            {"$push": {"ratings": rating}}
+        )
+
+
+def count_labeled_images(annotator):
+    """Count how many images a person has labeled based on the 'ratings' field."""
+    pipeline = [
+        {
+            "$match": {
+                "ratings.rater": annotator  # where 'rater' is the given annotator
+            }
+        },
+        {
+            "$count": "labeled_images"  # count the number of documents that match
+        }
+    ]
+
+    result = list(collection.aggregate(pipeline))
+
+    return result[0]['labeled_images'] if result else 0
+
+
+def prepare_next_image(annotator):
+    """Fetch the next image and its metadata."""
+    entry = fetch_random_entry(annotator)
+
+    if not entry:
+        return None, None, None, None, "No more images to rate!", None
+
+    meta_info = entry["meta_information"]
+    input_image = download_image(meta_info["input_img_link"])
+    output_image = download_image(meta_info["output_img_link"])
+    instruction = meta_info["instruction"]
+    progress_message = f"Rate this image! ({count_labeled_images(annotator)}/528 labeled)"
+
+    return meta_info["id"], input_image, output_image, instruction, progress_message, meta_info["turn"]
+
+
+def start(annotator):
+    return prepare_next_image(annotator)
+
+
+def record_input(id, turn, annotator, alignment, quality, consistency, overall):
+    save_rating(id, turn, annotator, alignment, quality, consistency, overall)
+    return prepare_next_image(annotator)
+
+
+# Gradio Interface
+def create_interface():
+    with gr.Blocks() as demo:
+        gr.Markdown(instruction_beginning)
+
+        # annotator = gr.Textbox(label="Nickname", interactive=True)
+        annotator = gr.Textbox(label="Annotator Nickname")
+
+        start_btn = gr.Button("Start")
+        progress_text = gr.Markdown("Waiting to start.")
+        # progress_text = gr.Markdown("You have labeled **0** out of 528 potential images.")
+
+        with gr.Row():
+            img_block1 = gr.Image(visible=True, width=300, height=300, label="Original Image", interactive=False)
+            img_block2 = gr.Image(visible=True, width=300, height=300, label="Edited Image", interactive=False)
+
+        prompt = gr.Textbox(label="Instruction", visible=True, interactive=False)
+        img_id = gr.Textbox(visible=False)
+        turn = gr.Textbox(visible=False)
+
+        with gr.Row():
+            slider_alignment = gr.Slider(label="Instruction-Edit Alignment", minimum=0, maximum=10, step=1, value=5,
+                                         info=alignment_info)
+            slider_quality = gr.Slider(label="Visual Quality", minimum=0, maximum=10, step=1, value=5,
+                                       info=quality_info)
+            slider_consistency = gr.Slider(label="Consistency", minimum=0, maximum=10, step=1, value=5,
+                                           info=consistency_info)
+
+        slider_overall = gr.Slider(label="Overall Impression", minimum=0, maximum=10, step=1, value=5,
+                                   info=overall_info)
+
+        save_and_continue_btn = gr.Button("Save & Continue")
+
+        start_btn.click(
+            fn=start,
+            inputs=[annotator],
+            outputs=[img_id, img_block1, img_block2, prompt, progress_text, turn]
+        )
+
+        save_and_continue_btn.click(
+            fn=record_input,
+            inputs=[img_id, turn, annotator, slider_alignment, slider_quality, slider_consistency, slider_overall],
+            outputs=[img_id, img_block1, img_block2, prompt, progress_text, turn]
+        )
+
+    return demo
+
+
+if __name__ == "__main__":
+    demo = create_interface()
+    demo.queue()
+    demo.launch(share=True, debug=True, auth=(gradio_user, gradio_password))

mongodb.ipynb

@@ -0,0 +1,236 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Create db structure"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import re"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2025-01-07T19:49:05.706326Z",
+     "start_time": "2025-01-07T19:49:05.697814Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "outputs": [],
+   "source": [
+    "base_url = \"https://raw.githubusercontent.com/piadonabauer/magicbrush-dev/main/images\""
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2025-01-07T19:49:06.010163Z",
+     "start_time": "2025-01-07T19:49:06.006475Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-01-07T19:49:06.347366Z",
+     "start_time": "2025-01-07T19:49:06.324433Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data saved at mongo_init.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "output_data = []\n",
+    "\n",
+    "def extract_turn(output_filename):\n",
+    "    match = re.search(r\"output(\\d+)\", output_filename)\n",
+    "    return int(match.group(1)) if match else None\n",
+    "\n",
+    "with open(\"edit_sessions.json\", \"r\") as file:\n",
+    "    edit_sessions = json.load(file)\n",
+    "\n",
+    "for id, sessions in edit_sessions.items():\n",
+    "    for session in sessions:\n",
+    "        input_link = f\"{base_url}/{id}/{session['input']}\"\n",
+    "        output_link = f\"{base_url}/{id}/{session['output']}\"\n",
+    "\n",
+    "        turn = extract_turn(session['output'])\n",
+    "        if turn is None:\n",
+    "            print(f\"No turn value found in {session['output']} - skip.\")\n",
+    "            continue\n",
+    "\n",
+    "        document = {\n",
+    "            \"meta_information\": {\n",
+    "                \"id\": int(id),\n",
+    "                \"turn\": int(turn),\n",
+    "                \"input_img_link\": input_link,\n",
+    "                \"output_img_link\": output_link,\n",
+    "                \"instruction\": session[\"instruction\"]\n",
+    "            },\n",
+    "            \"ratings\": []\n",
+    "        }\n",
+    "        output_data.append(document)\n",
+    "\n",
+    "output_json_path = \"mongo_init.json\"\n",
+    "with open(output_json_path, \"w\") as outfile:\n",
+    "    json.dump(output_data, outfile, indent=4)\n",
+    "\n",
+    "print(f\"Data saved at {output_json_path}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Upload structure to mongo db"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "outputs": [],
+   "source": [
+    "from pymongo import MongoClient\n",
+    "from dotenv import load_dotenv\n",
+    "import os"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2025-01-07T19:49:07.701178Z",
+     "start_time": "2025-01-07T19:49:07.700478Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "4.8.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pymongo\n",
+    "print(pymongo.__version__)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2025-01-07T19:49:07.987410Z",
+     "start_time": "2025-01-07T19:49:07.983489Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "outputs": [],
+   "source": [
+    "#os.environ.pop('MONGO_PASSWORD', None)\n",
+    "load_dotenv() # load gitignore\n",
+    "\n",
+    "mongo_user = os.getenv('MONGO_USER')\n",
+    "mongo_password = os.getenv('MONGO_PASSWORD')\n",
+    "cluster_url = os.getenv('MONGO_CLUSTER_URL')\n",
+    "#print(mongo_user, mongo_password, cluster_url)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2025-01-07T19:50:05.798336Z",
+     "start_time": "2025-01-07T19:50:05.794142Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data added.\n"
+     ]
+    }
+   ],
+   "source": [
+    "connection_url = f\"mongodb+srv://{mongo_user}:{mongo_password}@{cluster_url}\"\n",
+    "client = MongoClient(connection_url)\n",
+    "\n",
+    "db = client[\"thesis\"]\n",
+    "collection = db[\"labeling\"]\n",
+    "\n",
+    "with open(output_json_path, \"r\") as infile:\n",
+    "    documents = json.load(infile)\n",
+    "    collection.insert_many(documents)\n",
+    "\n",
+    "print(\"Data added.\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2025-01-07T19:50:08.311238Z",
+     "start_time": "2025-01-07T19:50:06.147140Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "name": "enviro",
+   "language": "python",
+   "display_name": "Python (enviro)"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}