Adding all the files.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vit-gpt2-image-captioning"]
+	path = vit-gpt2-image-captioning
+	url = https://huggingface.co/nlpconnect/vit-gpt2-image-captioning

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/eye_for_blind.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,78 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="HtmlUnknownTag" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="myValues">
+        <value>
+          <list size="7">
+            <item index="0" class="java.lang.String" itemvalue="nobr" />
+            <item index="1" class="java.lang.String" itemvalue="noembed" />
+            <item index="2" class="java.lang.String" itemvalue="comment" />
+            <item index="3" class="java.lang.String" itemvalue="noscript" />
+            <item index="4" class="java.lang.String" itemvalue="embed" />
+            <item index="5" class="java.lang.String" itemvalue="script" />
+            <item index="6" class="java.lang.String" itemvalue="style" />
+          </list>
+        </value>
+      </option>
+      <option name="myCustomValuesEnabled" value="true" />
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="42">
+            <item index="0" class="java.lang.String" itemvalue="absl-py" />
+            <item index="1" class="java.lang.String" itemvalue="networkx" />
+            <item index="2" class="java.lang.String" itemvalue="huggingface-hub" />
+            <item index="3" class="java.lang.String" itemvalue="PyYAML" />
+            <item index="4" class="java.lang.String" itemvalue="gast" />
+            <item index="5" class="java.lang.String" itemvalue="MarkupSafe" />
+            <item index="6" class="java.lang.String" itemvalue="numpy" />
+            <item index="7" class="java.lang.String" itemvalue="pyasn1" />
+            <item index="8" class="java.lang.String" itemvalue="requests" />
+            <item index="9" class="java.lang.String" itemvalue="Jinja2" />
+            <item index="10" class="java.lang.String" itemvalue="fsspec" />
+            <item index="11" class="java.lang.String" itemvalue="pyasn1-modules" />
+            <item index="12" class="java.lang.String" itemvalue="safetensors" />
+            <item index="13" class="java.lang.String" itemvalue="certifi" />
+            <item index="14" class="java.lang.String" itemvalue="keras" />
+            <item index="15" class="java.lang.String" itemvalue="urllib3" />
+            <item index="16" class="java.lang.String" itemvalue="itsdangerous" />
+            <item index="17" class="java.lang.String" itemvalue="Markdown" />
+            <item index="18" class="java.lang.String" itemvalue="sympy" />
+            <item index="19" class="java.lang.String" itemvalue="Flask" />
+            <item index="20" class="java.lang.String" itemvalue="blinker" />
+            <item index="21" class="java.lang.String" itemvalue="tokenizers" />
+            <item index="22" class="java.lang.String" itemvalue="libclang" />
+            <item index="23" class="java.lang.String" itemvalue="transformers" />
+            <item index="24" class="java.lang.String" itemvalue="google-auth-oauthlib" />
+            <item index="25" class="java.lang.String" itemvalue="Werkzeug" />
+            <item index="26" class="java.lang.String" itemvalue="h5py" />
+            <item index="27" class="java.lang.String" itemvalue="tensorboard-data-server" />
+            <item index="28" class="java.lang.String" itemvalue="packaging" />
+            <item index="29" class="java.lang.String" itemvalue="torch" />
+            <item index="30" class="java.lang.String" itemvalue="click" />
+            <item index="31" class="java.lang.String" itemvalue="tqdm" />
+            <item index="32" class="java.lang.String" itemvalue="termcolor" />
+            <item index="33" class="java.lang.String" itemvalue="regex" />
+            <item index="34" class="java.lang.String" itemvalue="mpmath" />
+            <item index="35" class="java.lang.String" itemvalue="typing_extensions" />
+            <item index="36" class="java.lang.String" itemvalue="cachetools" />
+            <item index="37" class="java.lang.String" itemvalue="charset-normalizer" />
+            <item index="38" class="java.lang.String" itemvalue="grpcio" />
+            <item index="39" class="java.lang.String" itemvalue="gTTS" />
+            <item index="40" class="java.lang.String" itemvalue="google-auth" />
+            <item index="41" class="java.lang.String" itemvalue="idna" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N801" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.10" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/eye_for_blind.iml" filepath="$PROJECT_DIR$/.idea/eye_for_blind.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/vit-gpt2-image-captioning" vcs="Git" />
+  </component>
+</project>

app.py

@@ -0,0 +1,50 @@
+import base64
+import tempfile
+import numpy as np
+import gradio as gr
+from gtts import gTTS
+import inference_script
+import vit_gpt2
+import os
+import warnings
+warnings.filterwarnings('ignore')
+
+
+def process_image_and_generate_output(image, model_selection):
+
+    if model_selection == ('Basic Model (Trained only for 15 epochs without any hyperparameter tuning, utilizing '
+                           'inception v3)'):
+        result = inference_script.evaluate(image)
+        pred_caption = ' '.join(result).rsplit(' ', 1)[0]
+        pred_caption = pred_caption.replace('<unk>', '')
+    elif model_selection == 'ViT-GPT2 (SOTA model for Image captioning)':
+        result = vit_gpt2.predict_step(image)
+        pred_caption = result[0]
+    else:
+        return "Invalid model selection", None
+
+    # Generate speech from the caption
+    tts = gTTS(text=pred_caption, lang='en', slow=False)
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.mp3') as temp_audio:
+        audio_file_path = temp_audio.name
+        tts.save(audio_file_path)
+
+    # Read the audio file
+    with open(audio_file_path, "rb") as f:
+        audio_content = f.read()
+
+    # Clean up the temporary audio file
+    os.unlink(audio_file_path)
+    return pred_caption, audio_content
+
+
+iface = gr.Interface(fn=process_image_and_generate_output,
+                     inputs=["image", gr.Radio(["Basic Model (Trained only for 15 epochs without any hyperparameter "
+                                                "tuning, utilizing inception v3)", "ViT-GPT2 (SOTA model for Image "
+                                                                                   "captioning)"], label="Choose "
+                                                                                                         "Model")],
+                     outputs=["text", "audio"],
+                     title="Eye For Blind | Image Captioning & TTS",
+                     description="To be added")
+
+iface.launch()

efb-requirements/decoder_50epoch_weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:974619e48b783c3ba43f38c6e00cff2948d2be3ad46487ad7b48a612f3ab72e2
+size 24349812

efb-requirements/encoder_50epoch_weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7552a644a862a01215983b7906696c11427eda4f18efe8cd6ab5cae555105b9
+size 2109280

efb-requirements/inception_v3.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c192f732860a2d28ec83c054d7e656717ff027693fb2d3ad2eded243f73cba5f
+size 88104664

flagged/log.csv

@@ -0,0 +1,2 @@
+image,Choose Model,output 0,output 1,flag,username,timestamp
+"{""path"":""flagged\\image\\c272d1784bee86f02dd8\\99679241_adc853a5c0.jpg"",""url"":""http://127.0.0.1:7860/file=C:\\Users\\krish\\AppData\\Local\\Temp\\gradio\\28021d595e5e6c88962f4a2deef9ec5c14e5a8a1\\99679241_adc853a5c0.jpg"",""size"":128165,""orig_name"":""99679241_adc853a5c0.jpg"",""mime_type"":""""}",vit_gpt2,,,,,2024-02-08 20:15:09.773370

inference_script.py

@@ -0,0 +1,140 @@
+import numpy as np
+
+import tensorflow as tf
+import keras
+from keras.models import Model
+
+
+class Encoder(Model):
+    def __init__(self, embed_dim):
+        super(Encoder, self).__init__()
+        self.dense = tf.keras.layers.Dense(embed_dim)
+
+    def call(self, features):
+        features = self.dense(features)
+        features = tf.keras.activations.relu(features)
+
+        return features
+
+
+class Attention_model(Model):
+    def __init__(self, units):
+        super(Attention_model, self).__init__()
+        self.W1 = tf.keras.layers.Dense(units)
+        self.W2 = tf.keras.layers.Dense(units)
+        self.V = tf.keras.layers.Dense(1)
+        self.units = units
+
+    def call(self, features, hidden):
+        hidden_with_time_axis = hidden[:, tf.newaxis]
+
+        score = tf.keras.activations.tanh(self.W1(features) + self.W2(hidden_with_time_axis))
+
+        attention_weights = tf.keras.activations.softmax(self.V(score), axis=1)
+
+        context_vector = attention_weights * features
+
+        context_vector = tf.reduce_sum(context_vector, axis=1)
+
+        return context_vector, attention_weights
+
+
+class Decoder(Model):
+    def __init__(self, embed_dim, units, vocab_size):
+        super(Decoder, self).__init__()
+        self.units = units
+        self.attention = Attention_model(self.units)
+        self.embed = tf.keras.layers.Embedding(vocab_size, embed_dim)
+        self.gru = tf.keras.layers.GRU(self.units, return_sequences=True, return_state=True,
+                                       recurrent_initializer='glorot_uniform')
+        self.d1 = tf.keras.layers.Dense(self.units)
+        self.d2 = tf.keras.layers.Dense(vocab_size)
+
+    def call(self, x, features, hidden):
+        context_vector, attention_weights = self.attention(features, hidden)
+        embed = self.embed(x)
+        embed = tf.concat([tf.expand_dims(context_vector, 1), embed], axis=-1)
+        output, state = self.gru(embed)
+        output = self.d1(output)
+        output = tf.reshape(output, (-1, output.shape[2]))
+        output = self.d2(output)
+
+        return output, state, attention_weights
+
+    def init_state(self, batch_size):
+        return tf.zeros((batch_size, self.units))
+
+    def reset_state(self, batch_size):
+        return tf.zeros((batch_size, self.units))
+
+
+# Loading the tokenizer
+with open("efb-requirements/tokenizer.json", 'r', encoding='utf-8') as f:
+    loaded_tokenizer_json = f.read()
+
+tokenizer = tf.keras.preprocessing.text.tokenizer_from_json(loaded_tokenizer_json)
+
+
+def load_and_process_image(image, target_size=(299, 299)):
+    img = tf.convert_to_tensor(image)
+    img = tf.cast(img, tf.uint8)
+    img = tf.image.resize(img, target_size)
+    img = tf.keras.applications.inception_v3.preprocess_input(img)
+    return img
+
+
+image_features_extract_model = keras.models.load_model("efb-requirements/inception_v3.h5")
+
+embedding_dim = 256
+units = 512
+vocab_size = 5001
+
+encoder = Encoder(embedding_dim)
+decoder = Decoder(embedding_dim, units, vocab_size)
+
+# Creating dummy inputs
+dummy_img_input = tf.ones((32, 64, 2048))
+features = encoder(dummy_img_input)
+hidden = decoder.init_state(32)
+dec_input = tf.expand_dims([tokenizer.word_index['<start>']] * 32, 1)
+dec = decoder(dec_input, features, hidden)
+
+# Loading saved models
+encoder.load_weights("efb-requirements/encoder_50epoch_weights.h5")
+decoder.load_weights("efb-requirements/decoder_50epoch_weights.h5")
+
+
+def evaluate(image):
+    max_length = 39
+    attention_plot = np.zeros((max_length, 64))
+
+    hidden = decoder.reset_state(batch_size=1)
+    # processing the input image to desired format before extracting features
+    temp_input = tf.expand_dims(load_and_process_image(image), 0)
+
+    img_tensor_val = image_features_extract_model(temp_input)
+    img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))
+
+    cnn_features = encoder(img_tensor_val)
+
+    decoder_input = tf.expand_dims([tokenizer.word_index['<start>']], 0)
+
+    result = []
+    predictions = ''
+    for i in range(max_length):
+        predictions, hidden, attention_weights = decoder(decoder_input, cnn_features, hidden)
+
+        attention_plot[i] = tf.reshape(attention_weights, (-1,)).numpy()
+
+        predicted_id = tf.argmax(predictions[0]).numpy()
+        result.append(tokenizer.index_word[predicted_id])
+
+        if tokenizer.index_word[predicted_id] == '<end>':
+            # return result, attention_plot, predictions
+            return result
+
+        decoder_input = tf.expand_dims([predicted_id], 0)
+
+    attention_plot = attention_plot[:len(result), :]
+    # return result, attention_plot, predictions
+    return result

test.py

@@ -0,0 +1,50 @@
+import base64
+import tempfile
+import numpy as np
+import gradio as gr
+from gtts import gTTS
+import inference_script
+import vit_gpt2
+import os
+import warnings
+warnings.filterwarnings('ignore')
+
+
+def process_image_and_generate_output(image, model_selection):
+
+    if model_selection == ('Basic Model (Trained only for 15 epochs without any hyperparameter tuning, utilizing '
+                           'inception v3)'):
+        result = inference_script.evaluate(image)
+        pred_caption = ' '.join(result).rsplit(' ', 1)[0]
+        pred_caption = pred_caption.replace('<unk>', '')
+    elif model_selection == 'ViT-GPT2 (SOTA model for Image captioning)':
+        result = vit_gpt2.predict_step(image)
+        pred_caption = result[0]
+    else:
+        return "Invalid model selection", None
+
+    # Generate speech from the caption
+    tts = gTTS(text=pred_caption, lang='en', slow=False)
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.mp3') as temp_audio:
+        audio_file_path = temp_audio.name
+        tts.save(audio_file_path)
+
+    # Read the audio file
+    with open(audio_file_path, "rb") as f:
+        audio_content = f.read()
+
+    # Clean up the temporary audio file
+    os.unlink(audio_file_path)
+    return pred_caption, audio_content
+
+
+iface = gr.Interface(fn=process_image_and_generate_output,
+                     inputs=["image", gr.Radio(["Basic Model (Trained only for 15 epochs without any hyperparameter "
+                                                "tuning, utilizing inception v3)", "ViT-GPT2 (SOTA model for Image "
+                                                                                   "captioning)"], label="Choose "
+                                                                                                         "Model")],
+                     outputs=["text", "audio"],
+                     title="Eye For Blind | Image Captioning & TTS",
+                     description="Generate a caption for the uploaded image and convert it to speech.")
+
+iface.launch()

vit-gpt2-image-captioning

@@ -0,0 +1 @@
+Subproject commit dc68f91c06a1ba6f15268e5b9c13ae7a7c514084

vit_gpt2.py

@@ -0,0 +1,26 @@
+import tensorflow as tf
+from transformers import TFVisionEncoderDecoderModel, ViTImageProcessor, AutoTokenizer
+from PIL import Image
+
+model = TFVisionEncoderDecoderModel.from_pretrained("vit-gpt2-image-captioning", from_pt=True)
+feature_extractor = ViTImageProcessor.from_pretrained("vit-gpt2-image-captioning")
+tokenizer = AutoTokenizer.from_pretrained("vit-gpt2-image-captioning")
+
+max_length = 16
+num_beams = 4
+gen_kwargs = {"max_length": max_length, "num_beams": num_beams}
+
+
+def predict_step(img_array):
+    i_image = Image.fromarray(img_array)
+
+    if i_image.mode != "RGB":
+        i_image = i_image.convert(mode="RGB")
+
+    pixel_values = feature_extractor(images=i_image, return_tensors="tf", do_normalize=True).pixel_values
+
+    output_ids = model.generate(pixel_values, **gen_kwargs)
+
+    pred = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
+    pred = [p.strip() for p in pred]
+    return pred