Duplicate from taneemishere/html-code-generation-from-images-with-deep-neural-networks

.gitattributes

@@ -0,0 +1,31 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/html-code-generation-from-images-with-deep-neural-networks.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,89 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="160" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="63">
+            <item index="0" class="java.lang.String" itemvalue="traitlets" />
+            <item index="1" class="java.lang.String" itemvalue="functools32" />
+            <item index="2" class="java.lang.String" itemvalue="protobuf" />
+            <item index="3" class="java.lang.String" itemvalue="html5lib" />
+            <item index="4" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="5" class="java.lang.String" itemvalue="PyYAML" />
+            <item index="6" class="java.lang.String" itemvalue="pickleshare" />
+            <item index="7" class="java.lang.String" itemvalue="python-dateutil" />
+            <item index="8" class="java.lang.String" itemvalue="cycler" />
+            <item index="9" class="java.lang.String" itemvalue="ipython-genutils" />
+            <item index="10" class="java.lang.String" itemvalue="tables" />
+            <item index="11" class="java.lang.String" itemvalue="appdirs" />
+            <item index="12" class="java.lang.String" itemvalue="Pygments" />
+            <item index="13" class="java.lang.String" itemvalue="scandir" />
+            <item index="14" class="java.lang.String" itemvalue="bleach" />
+            <item index="15" class="java.lang.String" itemvalue="pyparsing" />
+            <item index="16" class="java.lang.String" itemvalue="Markdown" />
+            <item index="17" class="java.lang.String" itemvalue="tensorflow-gpu" />
+            <item index="18" class="java.lang.String" itemvalue="Werkzeug" />
+            <item index="19" class="java.lang.String" itemvalue="h5py" />
+            <item index="20" class="java.lang.String" itemvalue="unity-lens-photos" />
+            <item index="21" class="java.lang.String" itemvalue="pkg-resources" />
+            <item index="22" class="java.lang.String" itemvalue="python-gflags" />
+            <item index="23" class="java.lang.String" itemvalue="leveldb" />
+            <item index="24" class="java.lang.String" itemvalue="pexpect" />
+            <item index="25" class="java.lang.String" itemvalue="Theano" />
+            <item index="26" class="java.lang.String" itemvalue="matplotlib" />
+            <item index="27" class="java.lang.String" itemvalue="virtualenv" />
+            <item index="28" class="java.lang.String" itemvalue="mock" />
+            <item index="29" class="java.lang.String" itemvalue="Keras" />
+            <item index="30" class="java.lang.String" itemvalue="enum34" />
+            <item index="31" class="java.lang.String" itemvalue="numexpr" />
+            <item index="32" class="java.lang.String" itemvalue="scikit-image" />
+            <item index="33" class="java.lang.String" itemvalue="pbr" />
+            <item index="34" class="java.lang.String" itemvalue="decorator" />
+            <item index="35" class="java.lang.String" itemvalue="networkx" />
+            <item index="36" class="java.lang.String" itemvalue="ptyprocess" />
+            <item index="37" class="java.lang.String" itemvalue="funcsigs" />
+            <item index="38" class="java.lang.String" itemvalue="backports.shutil-get-terminal-size" />
+            <item index="39" class="java.lang.String" itemvalue="wcwidth" />
+            <item index="40" class="java.lang.String" itemvalue="numpy" />
+            <item index="41" class="java.lang.String" itemvalue="simplegeneric" />
+            <item index="42" class="java.lang.String" itemvalue="adium-theme-ubuntu" />
+            <item index="43" class="java.lang.String" itemvalue="ipdb" />
+            <item index="44" class="java.lang.String" itemvalue="backports.weakref" />
+            <item index="45" class="java.lang.String" itemvalue="PyWavelets" />
+            <item index="46" class="java.lang.String" itemvalue="prompt-toolkit" />
+            <item index="47" class="java.lang.String" itemvalue="Cython" />
+            <item index="48" class="java.lang.String" itemvalue="nose" />
+            <item index="49" class="java.lang.String" itemvalue="scipy" />
+            <item index="50" class="java.lang.String" itemvalue="subprocess32" />
+            <item index="51" class="java.lang.String" itemvalue="tensorflow-tensorboard" />
+            <item index="52" class="java.lang.String" itemvalue="six" />
+            <item index="53" class="java.lang.String" itemvalue="opencv-python" />
+            <item index="54" class="java.lang.String" itemvalue="ipython" />
+            <item index="55" class="java.lang.String" itemvalue="packaging" />
+            <item index="56" class="java.lang.String" itemvalue="futures" />
+            <item index="57" class="java.lang.String" itemvalue="pathlib2" />
+            <item index="58" class="java.lang.String" itemvalue="pandas" />
+            <item index="59" class="java.lang.String" itemvalue="olefile" />
+            <item index="60" class="java.lang.String" itemvalue="Pyste" />
+            <item index="61" class="java.lang.String" itemvalue="pytz" />
+            <item index="62" class="java.lang.String" itemvalue="Pillow" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="model.compclasses.Compiler.compclasses" />
+          <option value="model.compclasses.Utils.Utils.get_preprocessed_img" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (dnn)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/html-code-generation-from-images-with-deep-neural-networks.iml" filepath="$PROJECT_DIR$/.idea/html-code-generation-from-images-with-deep-neural-networks.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

README.md

@@ -0,0 +1,14 @@
+---
+title: Image to HTML Code Demo
+emoji: 🧑‍💻
+colorFrom: pink
+colorTo: purple
+sdk: gradio
+sdk_version: 3.1.4
+app_file: app.py
+pinned: false
+license: afl-3.0
+duplicated_from: taneemishere/html-code-generation-from-images-with-deep-neural-networks
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,54 @@
+__author__ = 'Taneem Jan, taneemishere.github.io'
+
+import gradio as gr
+import main_program
+
+
+# our model's i/o method that take image from gradio interface's inputs.Image()
+def model_interface(image):
+    return main_model(image)
+
+
+# main method that call the main_program where code is generated and then compiled
+def main_model(input_image):
+    result = main_program.main_method(input_image)
+    return result
+
+
+interface_title = "<br> <p style='margin: 0% 8% 0% 8%'>HTML Code Generation from Images with Deep Neural Networks</p>"
+interface_description = """<p style='margin: 0% 8% 2% 8%; text-align: justify;text-justify: inter-word;'> Writing 
+code in a programming language for a designed mockup or a graphical user interface created by designers and UI 
+engineers, is done mostly by developers to build and develop custom websites and software. The development work is 
+not approachable by those unfamiliar with programming, to drive these personas capable of designing and developing 
+the code bases and website structures we come up with an automated system. In this work, we showed and proposed that 
+methods of deep learning and computer vision can be grasped to train a model that will automatically generate HTML 
+code from a single input mockup image and try to build an end-to-end automated system with accuracy more than 
+previous works for developing the structures of web pages.</p> """
+
+interface_article = """<br><h2 style='text-align: center;'>Limitations of Model</h2> <p style='text-align: 
+center;'>Certain limitations are there in the model some of them are listed below</p> <ul><li>Sometimes the model do 
+produce all the buttons with the same green color instead of other colors</li><li>As the model has fed with the data 
+provided, and so while producing the code on some other types of images might not generate the code we 
+wanted</li><li>The model is only trained upon the learning and recognition of boxes and buttons etc. in the images 
+and it do not write the text written exactly on the images</li></ul> 
+<div style='text-align: center;'> <br><br><a 
+href='https://twitter.com/taneemishere' target='_blank'>Developed by Taneem Jan</a> </div> <div style='text-align: 
+center;'> <a href='https://taneemishere.github.io/projects/project-one.html' target='_blank'>Paper</a> &ensp; &emsp; 
+<a href='https://github.com/taneemishere/html-code-generation-from-images-with-deep-neural-networks' 
+target='_blank'>Code</a> </div> """
+
+interface_examples = ['examples/example-1.png', 'examples/example-2.png', 'examples/example-3.png']
+
+# a gradio interface to convert a image to HTML Code
+interface = gr.Interface(
+    model_interface,
+    inputs='image',
+    outputs='text',
+    allow_flagging="manual",
+    title=interface_title,
+    description=interface_description,
+    article=interface_article,
+    examples=interface_examples
+)
+
+interface.launch(share=False)

classes/Sampler.py

@@ -0,0 +1,59 @@
+from __future__ import print_function
+from __future__ import absolute_import
+__author__ = 'Taneem Jan, taneemishere.github.io'
+
+from .Vocabulary import *
+from .Utils import *
+
+
+class Sampler:
+    def __init__(self, voc_path, input_shape, output_size, context_length):
+        self.voc = Vocabulary()
+        self.voc.retrieve(voc_path)
+
+        self.input_shape = input_shape
+        self.output_size = output_size
+
+        print("Vocabulary size: {}".format(self.voc.size))
+        print("Input shape: {}".format(self.input_shape))
+        print("Output size: {}".format(self.output_size))
+
+        self.context_length = context_length
+
+    def predict_greedy(self, model, input_img, require_sparse_label=True, sequence_length=150, verbose=False):
+        current_context = [self.voc.vocabulary[PLACEHOLDER]] * (self.context_length - 1)
+        current_context.append(self.voc.vocabulary[START_TOKEN])
+        if require_sparse_label:
+            current_context = Utils.sparsify(current_context, self.output_size)
+
+        predictions = START_TOKEN
+        out_probas = []
+
+        for i in range(0, sequence_length):
+            if verbose:
+                print("predicting {}/{}...".format(i, sequence_length))
+
+            probas = model.predict(input_img, np.array([current_context]))
+            prediction = np.argmax(probas)
+            out_probas.append(probas)
+
+            new_context = []
+            for j in range(1, self.context_length):
+                new_context.append(current_context[j])
+
+            if require_sparse_label:
+                sparse_label = np.zeros(self.output_size)
+                sparse_label[prediction] = 1
+                new_context.append(sparse_label)
+            else:
+                new_context.append(prediction)
+
+            current_context = new_context
+
+            predictions += self.voc.token_lookup[prediction]
+
+            if self.voc.token_lookup[prediction] == END_TOKEN:
+                break
+
+        return predictions, out_probas
+

classes/Utils.py

@@ -0,0 +1,39 @@
+__author__ = 'Taneem Jan, taneemishere.github.io'
+
+import numpy as np
+
+
+class Utils:
+    @staticmethod
+    def sparsify(label_vector, output_size):
+        sparse_vector = []
+
+        for label in label_vector:
+            sparse_label = np.zeros(output_size)
+            sparse_label[label] = 1
+
+            sparse_vector.append(sparse_label)
+
+        return np.array(sparse_vector)
+
+    @staticmethod
+    def get_preprocessed_img(img_path, image_size):
+        import cv2
+        # from keras.preprocessing.image import array_to_img, img_to_array
+        # img = array_to_img(img_path)
+        # img = img_to_array(img)
+        # img = cv2.imread(img_path)
+        # don't need to read the image as we're now directly passing the 
+        # image as numpy array to this method
+        img = cv2.resize(img_path, (image_size, image_size))
+        img = img.astype('float32')
+        img /= 255
+        return img
+
+    @staticmethod
+    def show(image):
+        import cv2
+        cv2.namedWindow("view", cv2.WINDOW_AUTOSIZE)
+        cv2.imshow("view", image)
+        cv2.waitKey(0)
+        cv2.destroyWindow("view")

classes/Vocabulary.py

@@ -0,0 +1,78 @@
+__author__ = 'Taneem Jan, taneemishere.github.io'
+
+import sys
+import numpy as np
+
+START_TOKEN = "<START>"
+END_TOKEN = "<END>"
+PLACEHOLDER = " "
+SEPARATOR = '->'
+
+
+class Vocabulary:
+    def __init__(self):
+        self.binary_vocabulary = {}
+        self.vocabulary = {}
+        self.token_lookup = {}
+        self.size = 0
+
+        self.append(START_TOKEN)
+        self.append(END_TOKEN)
+        self.append(PLACEHOLDER)
+
+    def append(self, token):
+        if token not in self.vocabulary:
+            self.vocabulary[token] = self.size
+            self.token_lookup[self.size] = token
+            self.size += 1
+
+    def create_binary_representation(self):
+        if sys.version_info >= (3,):
+            items = self.vocabulary.items()
+        else:
+            items = self.vocabulary.iteritems()
+        for key, value in items:
+            binary = np.zeros(self.size)
+            binary[value] = 1
+            self.binary_vocabulary[key] = binary
+
+    def get_serialized_binary_representation(self):
+        if len(self.binary_vocabulary) == 0:
+            self.create_binary_representation()
+
+        string = ""
+        if sys.version_info >= (3,):
+            items = self.binary_vocabulary.items()
+        else:
+            items = self.binary_vocabulary.iteritems()
+        for key, value in items:
+            array_as_string = np.array2string(value, separator=',', max_line_width=self.size * self.size)
+            string += "{}{}{}\n".format(key, SEPARATOR, array_as_string[1:len(array_as_string) - 1])
+        return string
+
+    def save(self, path):
+        output_file_name = "{}/words.vocab".format(path)
+        output_file = open(output_file_name, 'w')
+        output_file.write(self.get_serialized_binary_representation())
+        output_file.close()
+
+    def retrieve(self, path):
+        input_file = open("{}/words.vocab".format(path), 'r')
+        buffer = ""
+        for line in input_file:
+            try:
+                separator_position = len(buffer) + line.index(SEPARATOR)
+                buffer += line
+                key = buffer[:separator_position]
+                value = buffer[separator_position + len(SEPARATOR):]
+                value = np.fromstring(value, sep=',')
+
+                self.binary_vocabulary[key] = value
+                self.vocabulary[key] = np.where(value == 1)[0][0]
+                self.token_lookup[np.where(value == 1)[0][0]] = key
+
+                buffer = ""
+            except ValueError:
+                buffer += line
+        input_file.close()
+        self.size = len(self.vocabulary)

classes/model/AModel.py

@@ -0,0 +1,25 @@
+__author__ = 'Taneem Jan, taneemishere.github.io'
+
+from keras.models import model_from_json
+
+
+class AModel:
+    def __init__(self, input_shape, output_size, output_path):
+        self.model = None
+        self.input_shape = input_shape
+        self.output_size = output_size
+        self.output_path = output_path
+        self.name = ""
+
+    def save(self):
+        model_json = self.model.to_json()
+        with open("{}/{}.json".format(self.output_path, self.name), "w") as json_file:
+            json_file.write(model_json)
+        self.model.save_weights("{}/{}.h5".format(self.output_path, self.name))
+
+    def load(self, name=""):
+        output_name = self.name if name == "" else name
+        with open("{}/{}.json".format(self.output_path, output_name), "r") as json_file:
+            loaded_model_json = json_file.read()
+        self.model = model_from_json(loaded_model_json)
+        self.model.load_weights("{}/{}.h5".format(self.output_path, output_name))

classes/model/Config.py

@@ -0,0 +1,7 @@
+__author__ = 'Taneem Jan, taneemishere.github.io'
+
+CONTEXT_LENGTH = 48
+IMAGE_SIZE = 256
+BATCH_SIZE = 64
+EPOCHS = 10
+STEPS_PER_EPOCH = 72000

classes/model/Main_Model.py

@@ -0,0 +1,71 @@
+__author__ = 'Taneem Jan, improved the old model through pretrained Auto-encoders'
+
+from keras.layers import Input, Dense, Dropout, RepeatVector, LSTM, concatenate, Flatten
+from keras.models import Sequential, Model
+from tensorflow.keras.optimizers import RMSprop
+from .Config import *
+from .AModel import *
+from .autoencoder_image import *
+
+
+class Main_Model(AModel):
+    def __init__(self, input_shape, output_size, output_path):
+        AModel.__init__(self, input_shape, output_size, output_path)
+        self.name = "Main_Model"
+
+        visual_input = Input(shape=input_shape)
+
+        # Load the pre-trained autoencoder model
+        autoencoder_model = autoencoder_image(input_shape, input_shape, output_path)
+        autoencoder_model.load('autoencoder')
+        path = "classes/model/bin/"
+        path_to_autoencoder = "{}autoencoder.h5".format(path)
+        autoencoder_model.model.load_weights(path_to_autoencoder)
+        # Get only the model up to the encoded part
+        hidden_layer_model_freeze = Model(
+            inputs=autoencoder_model.model.input,
+            outputs=autoencoder_model.model.get_layer('encoded_layer').output
+        )
+        hidden_layer_input = hidden_layer_model_freeze(visual_input)
+
+        # Additional layers before concatenation
+        hidden_layer_model = Flatten()(hidden_layer_input)
+        hidden_layer_model = Dense(1024, activation='relu')(hidden_layer_model)
+        hidden_layer_model = Dropout(0.3)(hidden_layer_model)
+        hidden_layer_model = Dense(1024, activation='relu')(hidden_layer_model)
+        hidden_layer_model = Dropout(0.3)(hidden_layer_model)
+        hidden_layer_result = RepeatVector(CONTEXT_LENGTH)(hidden_layer_model)
+
+        # Making sure the loaded hidden_layer_model_freeze will no longer be updated
+        for layer in hidden_layer_model_freeze.layers:
+            layer.trainable = False
+
+        # The same language model that of pix2code by Tony Beltramelli
+        language_model = Sequential()
+        language_model.add(LSTM(128, return_sequences=True, input_shape=(CONTEXT_LENGTH, output_size)))
+        language_model.add(LSTM(128, return_sequences=True))
+
+        textual_input = Input(shape=(CONTEXT_LENGTH, output_size))
+        encoded_text = language_model(textual_input)
+
+        decoder = concatenate([hidden_layer_result, encoded_text])
+
+        decoder = LSTM(512, return_sequences=True)(decoder)
+        decoder = LSTM(512, return_sequences=False)(decoder)
+        decoder = Dense(output_size, activation='softmax')(decoder)
+
+        self.model = Model(inputs=[visual_input, textual_input], outputs=decoder)
+
+        optimizer = RMSprop(learning_rate=0.0001, clipvalue=1.0)
+        self.model.compile(loss='categorical_crossentropy', optimizer=optimizer)
+
+    def fit_generator(self, generator, steps_per_epoch):
+        # self.model.summary()
+        self.model.fit_generator(generator, steps_per_epoch=steps_per_epoch, epochs=EPOCHS, verbose=1)
+        self.save()
+
+    def predict(self, image, partial_caption):
+        return self.model.predict([image, partial_caption], verbose=0)[0]
+
+    def predict_batch(self, images, partial_captions):
+        return self.model.predict([images, partial_captions], verbose=1)