Update files

README.md

@@ -1,10 +1,10 @@
 ---
-title: Spell Net
-emoji: 🐢
-colorFrom: red
-colorTo: yellow
+title: SpellNet
+emoji: 👀
+colorFrom: yellow
+colorTo: blue
 sdk: gradio
-sdk_version: 4.12.0
+sdk_version: 4.10.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -0,0 +1,158 @@
+import numpy as np
+import os
+import json
+from tensorflow.math import argmax
+import gradio as gr
+from tensorflow.keras.models import model_from_json
+import src.video_with_landmarks
+import src.video_to_landmark_coordinates
+import src.preprocess_coordinates_data
+from src.load_model import Embedding, Encoder, Decoder, LandmarkEmbedding, EncoderTransformerBlock, MultiHeadAttention, \
+    DecoderTransformerBlock
+import src.predict_sequence
+
+# Load the character to prediction index dictionary
+character_to_prediction = 'src/character_to_prediction_index.json'
+with open(character_to_prediction) as json_file:
+    ORD2CHAR = json.load(json_file)
+
+# Load the variables from the JSON file
+json_file_path = "src/variables.json"
+with open(json_file_path, 'r') as json_file:
+    variables_dict = json.load(json_file)
+
+# Load the model architecture from the JSON file
+json_file = open('src/model_architecture.json', 'r')
+loaded_model_json = json_file.read()
+json_file.close()
+
+# Import lips landmark indices
+LIPS_LANDMARK_IDXS = np.array(variables_dict['LIPS_LANDMARK_IDXS'])
+
+custom_objects = {'Embedding': Embedding,
+                  'Encoder': Encoder,
+                  'Decoder': Decoder,
+                  'LandmarkEmbedding': LandmarkEmbedding,
+                  'EncoderTransformerBlock': EncoderTransformerBlock,
+                  'MultiHeadAttention': MultiHeadAttention,
+                  'DecoderTransformerBlock': DecoderTransformerBlock}
+
+
+def process_and_print_sequence(df):
+    """
+    Process the input DataFrame using specified data processing steps and print the shapes of the sequences.
+
+    Parameters:
+    df (pd.DataFrame): Input DataFrame containing tracking data.
+
+    Returns:
+    processed_sequence (np.ndarray): Processed sequence as a NumPy array.
+    """
+    LEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = src.preprocess_coordinates_data.get_idxs(df, ['left_hand'], ['z'])
+    RIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = src.preprocess_coordinates_data.get_idxs(df, ['right_hand'], ['z'])
+    LIPS_IDXS0, LIPS_NAMES0 = src.preprocess_coordinates_data.get_idxs(df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)
+    COLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))
+    N_COLS0 = len(COLUMNS0)
+
+    df = df[COLUMNS0]  # select only columns of interest equal to N_COLS0
+    all_tracking_sequence = df.values.reshape(1, -1, N_COLS0).astype(
+        np.float32)  # reshape after converting DataFrame to numpy array
+    preprocess_layer_instance = src.preprocess_coordinates_data.PreprocessLayer()  # instantiate PreprocessLayer class
+    processed_sequence = preprocess_layer_instance(all_tracking_sequence)  # call instance with data
+
+    print(f'input sequence shape: {all_tracking_sequence.shape}')
+    print(f'processed sequence shape: {processed_sequence.shape}')
+
+    return processed_sequence
+
+
+def predict_final_sequence(processed_sequence, model):
+    """
+        This function makes a prediction on a given sequence using a pre-trained model.
+
+        The sequence is expanded along the 0th dimension to account for batch size.
+        The prediction is made using the `predict_phrase` function, which should return a one-hot encoded prediction.
+        This one-hot encoded prediction is then converted into index values using argmax.
+        Finally, these index values are converted into a string representation using the `outputs2phrase` function.
+
+        Args:
+        processed_sequence (numpy array): An array representing the sequence to make a prediction on.
+                                          This should be of shape (128,164).
+        model (tensorflow.python.keras.engine.training.Model): The pre-trained model to use for making predictions.
+
+        Returns:
+        final_prediction (str): The final prediction made by the model, represented as a string.
+        """
+    # change shape to (1,128,164)
+    sequence = np.expand_dims(processed_sequence, axis=0)  # change shape to (1,128,164)
+
+    # Convert the one-hot encoded prediction to a string
+    predicted_phrase_one_hot = src.predict_sequence.predict_phrase(sequence, model)
+    predicted_phrase_one_hot = predicted_phrase_one_hot[0]  # Remove the batch dimension
+    predicted_phrase = argmax(predicted_phrase_one_hot, axis=-1).numpy()  # Convert one-hot encoding to index values
+    print(predicted_phrase)
+    final_prediction = src.predict_sequence.outputs2phrase(predicted_phrase, ORD2CHAR)
+    return final_prediction
+
+
+def video_identity(video):
+    """
+    Processes a video, extracts landmarks, feeds them to a pre-trained model, and makes a prediction.
+
+    The processing pipeline consists of the following steps:
+    1. Process the video with landmarks.
+    2. Extract landmarks coordinates and save them into a DataFrame.
+    3. Preprocess the landmarks.
+    4. Load a pre-trained model.
+    5. Feed the preprocessed landmarks to the model and get a prediction.
+
+    Parameters:
+    video (str): Path to the video file.
+
+    Returns:
+    tuple: The path to the processed video with landmarks and the predicted outcome.
+    """
+    # 1. load video and process it with landmarks
+    original_video_path = video
+    output_path = "src/video_landmarks.mp4"
+    src.video_with_landmarks.process_video_with_landmarks(original_video_path, output_path)
+
+    # 2. extract landmarks coordinates
+    df = src.video_to_landmark_coordinates.video_to_landmarks(output_path,
+                                                              src.video_to_landmark_coordinates.generate_column_names())
+    # Save the DataFrame to a CSV file
+    # df.to_csv('landmarks.csv', index=False)
+
+    # 3. preprocess landmarks
+    # Read data from a CSV file
+    # df = pd.read_csv('landmarks2.csv')
+    # df.drop(['sequence_id'],axis = 1, inplace=True)
+    processed_sequence = process_and_print_sequence(df)
+
+    # 4. load model
+    # load model architecture from JSON file
+    model = model_from_json(loaded_model_json, custom_objects=custom_objects)
+
+    # load weights into the new model
+    model.load_weights("src/model.h5")
+
+    # 5. predict
+    prediction = predict_final_sequence(processed_sequence, model)
+    print(prediction)
+
+    return output_path, prediction
+
+
+iface = gr.Interface(video_identity,
+                     inputs=gr.Video(label="Upload your video"),  # Adding a label to the input
+                     outputs=[gr.Video(label="Processed video"), gr.Textbox(label="Predicted sequence")],
+                     # Adding labels to the outputs
+                     title="SpellNet",  # Adding a title
+                     # Adding a description
+                     description="This application analyzes your video input to interpret American Sign Language (ASL) gestures corresponding to letters, numbers, and other signs. The output consists of the original video enhanced with overlaid landmarks that represent key points of ASL gestures, along with the predicted decoded ASL sequence expressed in textual form.",
+                     theme="gradio/monochrome",  # Changing the theme
+                     examples=[os.path.join(os.path.dirname(__file__), "src/videoplayback.mp4")],
+                     cache_examples=False)  # Disabling caching
+
+if __name__ == "__main__":
+    iface.launch(share=False)

requirements.txt

@@ -0,0 +1,6 @@
+
+opencv-python
+mediapipe
+tensorflow
+numpy
+gradio

src/character_to_prediction_index.json

@@ -0,0 +1,61 @@
+{
+    "0": " ",
+    "1": "!",
+    "2": "#",
+    "3": "$",
+    "4": "%",
+    "5": "&",
+    "6": "'",
+    "7": "(",
+    "8": ")",
+    "9": "*",
+    "10": "+",
+    "11": ",",
+    "12": "-",
+    "13": ".",
+    "14": "/",
+    "15": "0",
+    "16": "1",
+    "17": "2",
+    "18": "3",
+    "19": "4",
+    "20": "5",
+    "21": "6",
+    "22": "7",
+    "23": "8",
+    "24": "9",
+    "25": ":",
+    "26": ";",
+    "27": "=",
+    "28": "?",
+    "29": "@",
+    "30": "[",
+    "31": "_",
+    "32": "a",
+    "33": "b",
+    "34": "c",
+    "35": "d",
+    "36": "e",
+    "37": "f",
+    "38": "g",
+    "39": "h",
+    "40": "i",
+    "41": "j",
+    "42": "k",
+    "43": "l",
+    "44": "m",
+    "45": "n",
+    "46": "o",
+    "47": "p",
+    "48": "q",
+    "49": "r",
+    "50": "s",
+    "51": "t",
+    "52": "u",
+    "53": "v",
+    "54": "w",
+    "55": "x",
+    "56": "y",
+    "57": "z",
+    "58": "~"
+}

src/load_model.py

@@ -0,0 +1,346 @@
+import tensorflow as tf
+import json
+import numpy as np
+from tensorflow.keras.models import model_from_json
+import os
+
+# Convert the variables to the correct data type
+# Load the variables from the JSON file
+# Get the directory of the current file (preprocess_coordinates_data.py)
+current_directory = os.path.dirname(os.path.abspath(__file__))
+
+# Construct the path to variables.json
+json_file_path = os.path.join(current_directory, 'variables.json')
+with open(json_file_path, 'r') as json_file:
+    variables_dict = json.load(json_file)
+
+# Epsilon value for layer normalisation
+LAYER_NORM_EPS = variables_dict['LAYER_NORM_EPS']
+
+# final embedding and transformer embedding size
+UNITS_ENCODER = variables_dict['UNITS_ENCODER']
+UNITS_DECODER = variables_dict['UNITS_DECODER']
+
+# Transformer
+NUM_BLOCKS_ENCODER = variables_dict['NUM_BLOCKS_ENCODER']
+NUM_BLOCKS_DECODER = variables_dict['NUM_BLOCKS_DECODER']
+NUM_HEADS = variables_dict['NUM_HEADS']
+MLP_RATIO = variables_dict['MLP_RATIO']
+
+# Dropout
+EMBEDDING_DROPOUT = variables_dict['EMBEDDING_DROPOUT']
+MLP_DROPOUT_RATIO = variables_dict['MLP_DROPOUT_RATIO']
+MHA_DROPOUT_RATIO = variables_dict['MHA_DROPOUT_RATIO']
+CLASSIFIER_DROPOUT_RATIO = variables_dict['CLASSIFIER_DROPOUT_RATIO']
+
+# Number of Frames to resize recording to
+N_TARGET_FRAMES = variables_dict['N_TARGET_FRAMES']
+N_UNIQUE_CHARACTERS = variables_dict['N_UNIQUE_CHARACTERS']
+N_UNIQUE_CHARACTERS0 = variables_dict['N_UNIQUE_CHARACTERS0']
+PAD_TOKEN = variables_dict['PAD_TOKEN']
+SOS_TOKEN = variables_dict['SOS_TOKEN']
+
+# Length of Phrase + EOS Token
+MAX_PHRASE_LENGTH = variables_dict['MAX_PHRASE_LENGTH']
+
+# Mean/Standard Deviations of data used for normalizing
+MEANS = np.array(variables_dict['MEANS'])
+STDS = np.array(variables_dict['STDS'])
+
+# Initiailizers
+INIT_HE_UNIFORM = tf.keras.initializers.he_uniform
+INIT_GLOROT_UNIFORM = tf.keras.initializers.glorot_uniform
+INIT_ZEROS = tf.keras.initializers.constant(0.0)
+# Activations
+GELU = tf.keras.activations.gelu
+
+
+class Embedding(tf.keras.Model):
+    def __init__(self, **kwargs):
+        super(Embedding, self).__init__(**kwargs)
+        self.supports_masking = True
+
+    def build(self, input_shape):
+        self.positional_embedding = tf.Variable(
+            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_ENCODER], dtype=tf.float32),
+            trainable=True, name='embedding_positional_encoder')
+        self.dominant_hand_embedding = LandmarkEmbedding(UNITS_ENCODER, 'dominant_hand')
+
+    def call(self, x, training=False):
+        x = tf.where(tf.math.equal(x, 0.0), 0.0, (x - MEANS) / STDS)
+        x = self.dominant_hand_embedding(x)
+        x = x + self.positional_embedding
+        return x
+
+    def get_config(self):
+        return super().get_config()
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+class Encoder(tf.keras.Model):
+    def __init__(self, num_blocks, **kwargs):
+        super(Encoder, self).__init__(**kwargs)
+        self.num_blocks = num_blocks
+        self.support_masking = True
+        self.blocks = [
+            EncoderTransformerBlock(UNITS_ENCODER, NUM_HEADS, MLP_RATIO, MHA_DROPOUT_RATIO, MLP_DROPOUT_RATIO) for _ in
+            range(num_blocks)]
+
+        if UNITS_ENCODER != UNITS_DECODER:
+            self.dense_out = tf.keras.layers.Dense(UNITS_DECODER, kernel_initializer=INIT_GLOROT_UNIFORM,
+                                                   use_bias=False)
+            self.apply_dense_out = True
+        else:
+            self.apply_dense_out = False
+
+    def call(self, x, x_inp, training=False):
+        attention_mask = tf.where(tf.math.reduce_sum(x_inp, axis=[2]) == 0.0, 0.0, 1.0)
+        attention_mask = tf.expand_dims(attention_mask, axis=1)
+        attention_mask = tf.repeat(attention_mask, repeats=N_TARGET_FRAMES, axis=1)
+
+        for block in self.blocks:
+            x = block(x, attention_mask=attention_mask, training=training)
+
+        if self.apply_dense_out:
+            x = self.dense_out(x)
+
+        return x, attention_mask
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"num_blocks": self.num_blocks})
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+class Decoder(tf.keras.Model):
+    def __init__(self, num_blocks, **kwargs):
+        super(Decoder, self).__init__(**kwargs)
+        self.num_blocks = num_blocks
+        self.supports_masking = True
+        self.positional_embedding = tf.Variable(
+            initial_value=tf.zeros([N_TARGET_FRAMES, UNITS_DECODER], dtype=tf.float32),
+            trainable=True, name='embedding_positional_encoder')
+        self.char_emb = tf.keras.layers.Embedding(N_UNIQUE_CHARACTERS, UNITS_DECODER, embeddings_initializer=INIT_ZEROS)
+        self.pos_emb_mha = MultiHeadAttention(UNITS_DECODER, NUM_HEADS, MHA_DROPOUT_RATIO)
+        self.pos_emb_ln = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)
+        self.blocks = [
+            DecoderTransformerBlock(UNITS_DECODER, NUM_HEADS, MLP_RATIO, MHA_DROPOUT_RATIO, MLP_DROPOUT_RATIO) for _ in
+            range(num_blocks)]
+
+    def get_causal_attention_mask(self, B):
+        ones = tf.ones((N_TARGET_FRAMES, N_TARGET_FRAMES))
+        mask = tf.linalg.band_part(ones, 0, -1)
+        mask = tf.transpose(mask)
+        mask = tf.expand_dims(mask, axis=0)
+        mask = tf.tile(mask, [B, 1, 1])
+        mask = tf.cast(mask, tf.float32)
+        return mask
+
+    def call(self, encoder_outputs, attention_mask, phrase, training=False):
+        B = tf.shape(encoder_outputs)[0]
+        phrase = tf.cast(phrase, tf.int32)
+        phrase = tf.pad(phrase, [[0, 0], [1, 0]], constant_values=SOS_TOKEN, name='prepend_sos_token')
+        phrase = tf.pad(phrase, [[0, 0], [0, N_TARGET_FRAMES - MAX_PHRASE_LENGTH - 1]], constant_values=PAD_TOKEN,
+                        name='append_pad_token')
+        causal_mask = self.get_causal_attention_mask(B)
+        x = self.positional_embedding + self.char_emb(phrase)
+        x = self.pos_emb_ln(x + self.pos_emb_mha(x, x, x, attention_mask=causal_mask))
+
+        for block in self.blocks:
+            x = block(x, encoder_outputs, attention_mask=attention_mask, training=training)
+
+        x = tf.slice(x, [0, 0, 0], [-1, MAX_PHRASE_LENGTH, -1])
+        return x
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"num_blocks": self.num_blocks})
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+# Embeds a landmark using fully connected layers
+class LandmarkEmbedding(tf.keras.Model):
+    def __init__(self, units, name):
+        super(LandmarkEmbedding, self).__init__(name=f'{name}_embedding')
+        self.units = units
+        self.supports_masking = True
+
+    def build(self, input_shape):
+        # Embedding for missing landmark in frame, initizlied with zeros
+        self.empty_embedding = self.add_weight(
+            name=f'{self.name}_empty_embedding',
+            shape=[self.units],
+            initializer=INIT_ZEROS,
+        )
+        # Embedding
+        self.dense = tf.keras.Sequential([
+            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_1', use_bias=False,
+                                  kernel_initializer=INIT_GLOROT_UNIFORM, activation=GELU),  # Can change activation
+            tf.keras.layers.Dense(self.units, name=f'{self.name}_dense_2', use_bias=False,
+                                  kernel_initializer=INIT_HE_UNIFORM),
+        ], name=f'{self.name}_dense')
+
+    def call(self, x):
+        return tf.where(
+            # Checks whether landmark is missing in frame
+            tf.reduce_sum(x, axis=2, keepdims=True) == 0,
+            # If so, the empty embedding is used
+            self.empty_embedding,
+            # Otherwise the landmark data is embedded
+            self.dense(x),
+        )
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"units": self.units, "name": self.name})
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+class EncoderTransformerBlock(tf.keras.layers.Layer):
+    def __init__(self, units, num_heads, mlp_ratio, mha_dropout_ratio, mlp_dropout_ratio, **kwargs):
+        super(EncoderTransformerBlock, self).__init__(**kwargs)
+        self.layer_norm_1 = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)
+        self.mha = MultiHeadAttention(units, num_heads, mha_dropout_ratio)
+        self.layer_norm_2 = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)
+        self.mlp = tf.keras.Sequential([
+            tf.keras.layers.Dense(units * mlp_ratio, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM,
+                                  use_bias=False),
+            tf.keras.layers.Dropout(mlp_dropout_ratio),
+            tf.keras.layers.Dense(units, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),
+        ])
+
+    def call(self, inputs, attention_mask, training=False):
+        x = self.layer_norm_1(inputs + self.mha(inputs, inputs, inputs, attention_mask=attention_mask))
+        x = self.layer_norm_2(x + self.mlp(x))
+        return x
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"units": self.units, "num_heads": self.num_heads, "mlp_ratio": self.mlp_ratio,
+                       "mha_dropout_ratio": self.mha_dropout_ratio, "mlp_dropout_ratio": self.mlp_dropout_ratio})
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+# replaced softmax with softmax layer to support masked softmax
+def scaled_dot_product(q, k, v, softmax, attention_mask):
+    # calculates Q . K(transpose)
+    qkt = tf.matmul(q, k, transpose_b=True)
+    # calculates scaling factor
+    dk = tf.math.sqrt(tf.cast(q.shape[-1], dtype=tf.float32))
+    scaled_qkt = qkt / dk
+    softmax = softmax(scaled_qkt, mask=attention_mask)
+    z = tf.matmul(softmax, v)
+    # shape: (m,Tx,depth), same shape as q,k,v
+    return z
+
+
+class MultiHeadAttention(tf.keras.layers.Layer):
+    def __init__(self, d_model, num_of_heads, dropout, d_out=None):
+        super(MultiHeadAttention, self).__init__()
+        self.d_model = d_model
+        self.num_of_heads = num_of_heads
+        self.depth = d_model // num_of_heads  # Can change
+        self.wq = [tf.keras.layers.Dense(self.depth, use_bias=False) for i in
+                   range(num_of_heads)]  # depth//2 isn't common, we can try different numbers
+        self.wk = [tf.keras.layers.Dense(self.depth, use_bias=False) for i in range(num_of_heads)]
+        self.wv = [tf.keras.layers.Dense(self.depth, use_bias=False) for i in range(num_of_heads)]
+        self.softmax = tf.keras.layers.Softmax()
+        self.do = tf.keras.layers.Dropout(dropout)
+        self.supports_masking = True
+        self.wo = tf.keras.layers.Dense(d_model if d_out is None else d_out, use_bias=False)
+
+    def call(self, q, k, v, attention_mask=None, training=False):
+        multi_attn = []
+        for i in range(self.num_of_heads):
+            Q = self.wq[i](q)
+            K = self.wk[i](k)
+            V = self.wv[i](v)
+            multi_attn.append(scaled_dot_product(Q, K, V, self.softmax, attention_mask))
+
+        multi_head = tf.concat(multi_attn, axis=-1)
+        multi_head_attention = self.wo(multi_head)
+        multi_head_attention = self.do(multi_head_attention, training=training)
+
+        return multi_head_attention
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"d_model": self.d_model, "num_of_heads": self.num_of_heads, "dropout": self.dropout})
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+class DecoderTransformerBlock(tf.keras.layers.Layer):
+    def __init__(self, units, num_heads, mlp_ratio, mha_dropout_ratio, mlp_dropout_ratio, **kwargs):
+        super(DecoderTransformerBlock, self).__init__(**kwargs)
+        self.layer_norm_1 = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)
+        self.mha = MultiHeadAttention(units, num_heads, mha_dropout_ratio)
+        self.layer_norm_2 = tf.keras.layers.LayerNormalization(epsilon=LAYER_NORM_EPS)
+        self.mlp = tf.keras.Sequential([
+            tf.keras.layers.Dense(units * mlp_ratio, activation=GELU, kernel_initializer=INIT_GLOROT_UNIFORM,
+                                  use_bias=False),
+            tf.keras.layers.Dropout(mlp_dropout_ratio),
+            tf.keras.layers.Dense(units, kernel_initializer=INIT_HE_UNIFORM, use_bias=False),
+        ])
+
+    def call(self, inputs, encoder_outputs, attention_mask, training=False):
+        x = self.layer_norm_1(
+            inputs + self.mha(inputs, encoder_outputs, encoder_outputs, attention_mask=attention_mask))
+        x = self.layer_norm_2(x + self.mlp(x))
+        return x
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"units": self.units, "num_heads": self.num_heads, "mlp_ratio": self.mlp_ratio,
+                       "mha_dropout_ratio": self.mha_dropout_ratio, "mlp_dropout_ratio": self.mlp_dropout_ratio})
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
+
+custom_objects = {'Embedding': Embedding,
+                  'Encoder': Encoder,
+                  'Decoder': Decoder,
+                  'LandmarkEmbedding': LandmarkEmbedding,
+                  'EncoderTransformerBlock': EncoderTransformerBlock,
+                  'MultiHeadAttention': MultiHeadAttention,
+                  'DecoderTransformerBlock': DecoderTransformerBlock}
+
+# load json and create model
+model_archeticture_file_path = os.path.join(current_directory, 'model_architecture.json')
+json_file = open(model_archeticture_file_path, 'r')
+loaded_model_json = json_file.read()
+json_file.close()
+
+# load model from JSON file
+loaded_model = model_from_json(loaded_model_json, custom_objects=custom_objects)
+
+# load weights into the new model
+model_weights_file_path = os.path.join(current_directory, 'model.h5')
+loaded_model.load_weights(model_weights_file_path)
+
+# loaded_model.summary(expand_nested=True, show_trainable=True, )

src/model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d18e0fb837407b4b8f45411e48758a71d7ff149e31fec29b5868bc5b60e032ef
+size 27599904

src/model_architecture.json

@@ -0,0 +1 @@
+{"class_name": "Functional", "config": {"name": "model", "trainable": true, "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 128, 164], "dtype": "float32", "sparse": false, "ragged": false, "name": "frames"}, "name": "frames", "inbound_nodes": []}, {"class_name": "Masking", "config": {"name": "masking", "trainable": true, "dtype": "float32", "batch_input_shape": [null, 128, 164], "mask_value": 0.0}, "name": "masking", "inbound_nodes": [[["frames", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {}, "name": "embedding", "inbound_nodes": [[["masking", 0, 0, {}]]]}, {"class_name": "Encoder", "config": {"num_blocks": 3}, "name": "encoder", "inbound_nodes": [[["embedding", 0, 0, {"x_inp": ["frames", 0, 0]}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 32], "dtype": "int32", "sparse": false, "ragged": false, "name": "phrase"}, "name": "phrase", "inbound_nodes": []}, {"class_name": "Decoder", "config": {"num_blocks": 2}, "name": "decoder", "inbound_nodes": [[["encoder", 0, 0, {"attention_mask": ["encoder", 0, 1], "phrase": ["phrase", 0, 0]}]]]}, {"class_name": "Sequential", "config": {"name": "classifier", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 32, 384], "dtype": "float32", "sparse": false, "ragged": false, "name": "dropout_11_input"}}, {"class_name": "Dropout", "config": {"name": "dropout_11", "trainable": true, "dtype": "float32", "rate": 0.2, "noise_shape": null, "seed": null}}, {"class_name": "Dense", "config": {"name": "dense_88", "trainable": true, "dtype": "float32", "units": 62, "activation": "linear", "use_bias": false, "kernel_initializer": {"class_name": "HeUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "name": "classifier", "inbound_nodes": [[["decoder", 0, 0, {}]]]}], "input_layers": [["frames", 0, 0], ["phrase", 0, 0]], "output_layers": [["classifier", 1, 0]]}, "keras_version": "2.12.0", "backend": "tensorflow"}

src/predict_sequence.py

@@ -0,0 +1,89 @@
+import json
+import numpy as np
+import tensorflow as tf
+import os
+
+# Convert the variables to the correct data type
+# Load the variables from the JSON file
+# Get the directory of the current file (preprocess_coordinates_data.py)
+current_directory = os.path.dirname(os.path.abspath(__file__))
+
+# Construct the path to variables.json
+json_file_path = os.path.join(current_directory, 'variables.json')
+with open(json_file_path, 'r') as json_file:
+    variables_dict = json.load(json_file)
+
+MAX_PHRASE_LENGTH = variables_dict['MAX_PHRASE_LENGTH']
+PAD_TOKEN = variables_dict['PAD_TOKEN']
+N_UNIQUE_CHARACTERS = variables_dict['N_UNIQUE_CHARACTERS']
+
+# Read Character to Ordinal Encoding Mapping.
+character_to_prediction = os.path.join(current_directory, 'character_to_prediction_index.json')
+with open(character_to_prediction) as json_file:
+    ORD2CHAR = json.load(json_file)
+
+
+
+# Output Predictions to string
+def outputs2phrase(outputs, ORD2CHAR):
+    """
+    Convert output sequence to a human-readable phrase by mapping
+    each output to a corresponding character.
+
+    Parameters:
+    outputs (np.array): A sequence of model output,
+                        can be 1D (sequence of character IDs)
+                        or 2D (sequence of one-hot encodings).
+    ORD2CHAR (dict): A mapping from character IDs to characters.
+
+    Returns:
+    str: The converted phrase.
+    """
+    ORD2CHAR = {int(k): v for k, v in ORD2CHAR.items()} # Convert keys to integers
+    if outputs.ndim == 2:
+        outputs = np.argmax(outputs, axis=1)
+    return ''.join([ORD2CHAR.get(s, '') for s in outputs])
+
+
+@tf.function()
+def predict_phrase(batch_frames, model):
+    """
+    Use a pre-trained model to predict a phrase from a batch of frame sequences.
+
+    Parameters:
+    batch_frames (np.array): A batch of frame sequences.
+    model (tf.keras.Model): The pre-trained model to use for prediction.
+
+    Returns:
+    tf.Tensor: One-hot encoding of the predicted phrase.
+    """
+    batch_frames = tf.convert_to_tensor(batch_frames)
+    phrase = tf.fill([batch_frames.shape[0], MAX_PHRASE_LENGTH], PAD_TOKEN)
+    phrase = tf.cast(phrase, tf.int32)  # Cast phrase to int32 initially
+    for idx in tf.range(MAX_PHRASE_LENGTH):
+        # Predict Next Token
+        outputs = model({
+            'frames': batch_frames,
+            'phrase': phrase,
+        })
+
+        phrase = tf.where(
+            tf.range(MAX_PHRASE_LENGTH)[None, :] < idx + 1,
+            tf.argmax(outputs, axis=-1, output_type=tf.int32),
+            phrase,
+        )
+    # one-hot encode the outputs
+    outputs_one_hot = tf.one_hot(phrase, depth=N_UNIQUE_CHARACTERS)
+    return outputs_one_hot
+
+# # Assuming sequence is your array of shape (128, 164)
+# sequence = processed_sequence._shape(1, *processed_sequence.shape)  # reshapes sequence to (1, 128, 164)
+#
+# # Now you can feed sequence to your prediction function
+# pred_phrase_one_hot = predict_phrase(sequence)
+#
+# # Convert the one-hot encoded prediction to a string
+# # Remember the output is one-hot encoded so we need to convert it to integers first
+# pred_phrase = outputs2phrase(tf.argmax(pred_phrase_one_hot, axis=-1).numpy())
+#
+# print(pred_phrase)

src/preprocess_coordinates_data.py

@@ -0,0 +1,173 @@
+import pandas as pd
+import numpy as np
+import tensorflow as tf
+import json
+import os
+
+# Convert the variables to the correct data type
+# Load the variables from the JSON file
+current_directory = os.path.dirname(os.path.abspath(__file__))
+
+# Construct the path to variables.json
+json_file_path = os.path.join(current_directory, 'variables.json')
+with open(json_file_path, 'r') as json_file:
+    variables_dict = json.load(json_file)
+
+# Lips Landmark Face Ids
+LIPS_LANDMARK_IDXS = variables_dict['LIPS_LANDMARK_IDXS']
+N_TARGET_FRAMES = variables_dict['N_TARGET_FRAMES']
+N_DIMS0 = variables_dict['N_TARGET_FRAMES']
+
+# Read data from a CSV file
+csv_file_path = os.path.join(current_directory, 'landmarks.csv')
+
+df = pd.read_csv(csv_file_path)
+
+def get_idxs(df, words_pos, words_neg=[], ret_names=True, idxs_pos=None):
+    """
+    Given a DataFrame and a list of words, this function will find all the column names
+    that contain all the words in 'words_pos' and none of the words in 'words_neg'.
+
+    Parameters:
+        df (pandas.DataFrame): Dataframe to search for column names
+        words_pos (list of str): List of words that column names should contain
+        words_neg (list of str, optional): List of words that column names should not contain. Default is empty list.
+        ret_names (bool, optional): Whether to return column names. Default is True.
+        idxs_pos (list of int, optional): Column indices to search within. Default is None, which means search all columns.
+
+    Returns:
+        idxs (np.array): Column indices where column names meet the criteria
+        names (np.array): Column names that meet the criteria. Only returned if 'ret_names' is True.
+    """
+    idxs = []
+    names = []
+    for w in words_pos:
+        for col_idx, col in enumerate(df.columns):
+            # Exclude Non Landmark Columns
+            if col in ['frame']:
+                continue
+
+            col_idx = int(col.split('_')[-1])
+            # Check if column name contains all words
+            if (w in col) and (idxs_pos is None or col_idx in idxs_pos) and all([w not in col for w in words_neg]):
+                idxs.append(col_idx)
+                names.append(col)
+    # Convert to Numpy arrays
+    idxs = np.array(idxs)
+    names = np.array(names)
+    # Returns either both column indices and names
+    if ret_names:
+        return idxs, names
+    # Or only columns indices
+    else:
+        return idxs
+
+
+# Get the indices of columns of interest
+LEFT_HAND_IDXS0, LEFT_HAND_NAMES0 = get_idxs(df, ['left_hand'], ['z'])
+RIGHT_HAND_IDXS0, RIGHT_HAND_NAMES0 = get_idxs(df, ['right_hand'], ['z'])
+LIPS_IDXS0, LIPS_NAMES0 = get_idxs(df, ['face'], ['z'], idxs_pos=LIPS_LANDMARK_IDXS)
+COLUMNS0 = np.concatenate((LEFT_HAND_NAMES0, RIGHT_HAND_NAMES0, LIPS_NAMES0))
+N_COLS0 = len(COLUMNS0)
+N_COLS = N_COLS0
+
+
+class PreprocessLayerNonNaN(tf.keras.layers.Layer):
+    """
+    This is a custom layer in Keras that replaces NaN values in the input tensor with 0.
+    """
+
+    def __init__(self):
+        super(PreprocessLayerNonNaN, self).__init__()
+
+    @tf.function(
+        input_signature=(tf.TensorSpec(shape=[None, N_COLS0], dtype=tf.float32),),
+    )
+    def call(self, data0):
+        """
+        This method is called when the layer instance is called with some inputs.
+
+        Parameters:
+            data0 (Tensor): Input tensor
+
+        Returns:
+            data (Tensor): Output tensor with the same shape as the input, but with NaN values replaced with 0
+        """
+        # Fill NaN Values With 0
+        data = tf.where(tf.math.is_nan(data0), 0.0, data0)
+
+        # Hacky
+        data = data[None]
+
+        # Empty Hand Frame Filtering
+        hands = tf.slice(data, [0, 0, 0], [-1, -1, 84])
+        hands = tf.abs(hands)
+        mask = tf.reduce_sum(hands, axis=2)
+        mask = tf.not_equal(mask, 0)
+        data = data[mask][None]
+        data = tf.squeeze(data, axis=[0])
+
+        return data
+
+
+class PreprocessLayer(tf.keras.layers.Layer):
+    """
+    This is a custom layer in Keras that pre-processes the input data in a specific way,
+    which includes filling NaN values with 0, filtering empty frames and resizing frames.
+    """
+
+    def __init__(self):
+        super(PreprocessLayer, self).__init__()
+
+    @tf.function(
+        input_signature=(tf.TensorSpec(shape=[None, None, N_COLS0], dtype=tf.float32),),
+    )
+    def call(self, data0, resize=True):
+        """
+        This method is called when the layer instance is called with some inputs.
+
+        Parameters:
+            data0 (Tensor): Input tensor
+            resize (bool, optional): Whether to resize the frames. Default is True.
+
+        Returns:
+            data (Tensor): Output tensor after pre-processing
+        """
+        # Fill NaN Values With 0
+        data = tf.where(tf.math.is_nan(data0), 0.0, data0)
+
+        # Empty Hand Frame Filtering
+        hands = tf.slice(data, [0, 0, 0], [-1, -1, 84])
+        hands = tf.abs(hands)
+        mask = tf.reduce_sum(hands, axis=2)
+        mask = tf.not_equal(mask, 0)
+        data = data[mask][None]
+
+        # Pad Zeros
+        N_FRAMES = len(data[0])
+        if N_FRAMES < N_TARGET_FRAMES:
+            data = tf.concat((
+                data,
+                tf.zeros([1, N_TARGET_FRAMES - N_FRAMES, N_COLS], dtype=tf.float32)
+            ), axis=1)
+        # Downsample
+        data = tf.image.resize(
+            data,
+            [1, N_TARGET_FRAMES],
+            method=tf.image.ResizeMethod.BILINEAR,
+        )
+
+        # Squeeze Batch Dimension
+        data = tf.squeeze(data, axis=[0])
+
+        return data
+
+
+df = df[COLUMNS0]  # select only columns of interest equal to N_COLS0
+hand_tracking_sequence = df.values.reshape(1, -1, N_COLS0)  # reshape after converting DataFrame to numpy array
+
+preprocess_layer_instance = PreprocessLayer()  # instantiate PreprocessLayer class
+processed_sequence = preprocess_layer_instance(hand_tracking_sequence)  # call instance with data
+
+# print(f'input sequence shape: {hand_tracking_sequence.shape}')
+# print(f'processed sequence shape: {processed_sequence.shape}')

src/variables.json

@@ -0,0 +1,394 @@
+{
+    "LIPS_LANDMARK_IDXS": [
+        61,
+        185,
+        40,
+        39,
+        37,
+        0,
+        267,
+        269,
+        270,
+        409,
+        291,
+        146,
+        91,
+        181,
+        84,
+        17,
+        314,
+        405,
+        321,
+        375,
+        78,
+        191,
+        80,
+        81,
+        82,
+        13,
+        312,
+        311,
+        310,
+        415,
+        95,
+        88,
+        178,
+        87,
+        14,
+        317,
+        402,
+        318,
+        324,
+        308
+    ],
+    "LAYER_NORM_EPS": 1e-05,
+    "UNITS_ENCODER": 384,
+    "UNITS_DECODER": 384,
+    "NUM_BLOCKS_ENCODER": 3,
+    "NUM_BLOCKS_DECODER": 2,
+    "NUM_HEADS": 4,
+    "MLP_RATIO": 2,
+    "EMBEDDING_DROPOUT": 0.2,
+    "MLP_DROPOUT_RATIO": 0.2,
+    "MHA_DROPOUT_RATIO": 0.2,
+    "CLASSIFIER_DROPOUT_RATIO": 0.2,
+    "N_TARGET_FRAMES": 128,
+    "N_DIMS0": 2,
+    "N_UNIQUE_CHARACTERS": 62,
+    "N_UNIQUE_CHARACTERS0": 59,
+    "PAD_TOKEN": 59,
+    "SOS_TOKEN": 60,
+    "MAX_PHRASE_LENGTH": 32,
+    "MEANS": [
+        0.69352776,
+        0.60659605,
+        0.53412515,
+        0.4970676,
+        0.48584947,
+        0.5761701,
+        0.5300588,
+        0.49778917,
+        0.47764367,
+        0.6305243,
+        0.5822572,
+        0.55222154,
+        0.53908557,
+        0.68544865,
+        0.63951194,
+        0.6104323,
+        0.5991277,
+        0.7378051,
+        0.7018211,
+        0.6776119,
+        0.6673842,
+        0.76445776,
+        0.7457853,
+        0.7062872,
+        0.67484325,
+        0.6514734,
+        0.6304427,
+        0.5906848,
+        0.5854317,
+        0.5849309,
+        0.6276549,
+        0.5890438,
+        0.59771925,
+        0.6047316,
+        0.6383216,
+        0.60959125,
+        0.6295764,
+        0.6437836,
+        0.6588292,
+        0.6397078,
+        0.65018004,
+        0.65816236,
+        0.26357186,
+        0.35093567,
+        0.4236605,
+        0.45704976,
+        0.4634739,
+        0.37947592,
+        0.4234214,
+        0.45306972,
+        0.4717593,
+        0.3199842,
+        0.36261505,
+        0.38926786,
+        0.40241373,
+        0.26189587,
+        0.30273047,
+        0.3301876,
+        0.34255308,
+        0.20624675,
+        0.23920882,
+        0.263005,
+        0.27461466,
+        0.75472385,
+        0.73504084,
+        0.6943852,
+        0.6608657,
+        0.63613355,
+        0.6144105,
+        0.5700216,
+        0.56217206,
+        0.5597008,
+        0.611077,
+        0.56800383,
+        0.575002,
+        0.5811821,
+        0.62163454,
+        0.59134597,
+        0.61230445,
+        0.6277079,
+        0.64273566,
+        0.6216118,
+        0.6318555,
+        0.63973725,
+        0.56342137,
+        0.5647059,
+        0.5649758,
+        0.5657689,
+        0.54460865,
+        0.52689284,
+        0.51569146,
+        0.5043293,
+        0.51033896,
+        0.52668756,
+        0.53708506,
+        0.54991424,
+        0.5468167,
+        0.55006754,
+        0.5267238,
+        0.5178957,
+        0.51888436,
+        0.5099791,
+        0.53717476,
+        0.5305108,
+        0.5081805,
+        0.51886874,
+        0.58258605,
+        0.6024338,
+        0.6155048,
+        0.6306914,
+        0.6245343,
+        0.6058631,
+        0.59408224,
+        0.58018464,
+        0.5852319,
+        0.5804903,
+        0.60605526,
+        0.61589545,
+        0.61500907,
+        0.6246284,
+        0.59435004,
+        0.6024958,
+        0.6250273,
+        0.61513,
+        0.508501,
+        0.5193109,
+        0.52219623,
+        0.53701967,
+        0.5069547,
+        0.51169485,
+        0.51677644,
+        0.5253185,
+        0.5245756,
+        0.521367,
+        0.5199756,
+        0.51932734,
+        0.5365361,
+        0.5221106,
+        0.5230684,
+        0.53079647,
+        0.5238175,
+        0.52800494,
+        0.5223436,
+        0.5342269,
+        0.5212379,
+        0.52289945,
+        0.506347,
+        0.5106173,
+        0.51533395,
+        0.5235456,
+        0.5230225,
+        0.52027595,
+        0.51917976,
+        0.5189014,
+        0.5361387,
+        0.5216965,
+        0.5220167,
+        0.52960336,
+        0.5225625,
+        0.5264617,
+        0.5215638,
+        0.53341466,
+        0.51952803,
+        0.5216051
+    ],
+    "STDS": [
+        0.10834738,
+        0.10391748,
+        0.10296664,
+        0.10752504,
+        0.12336373,
+        0.10313869,
+        0.10744168,
+        0.11199072,
+        0.1193621,
+        0.10597368,
+        0.11260378,
+        0.1170811,
+        0.12447591,
+        0.11238337,
+        0.12130429,
+        0.12248141,
+        0.1267081,
+        0.1224081,
+        0.13301295,
+        0.13806877,
+        0.1437398,
+        0.08867608,
+        0.08839962,
+        0.08913112,
+        0.09358086,
+        0.09968524,
+        0.08439907,
+        0.09381164,
+        0.10565417,
+        0.11996002,
+        0.08592986,
+        0.1002507,
+        0.11805841,
+        0.13548768,
+        0.08893858,
+        0.1042807,
+        0.11806193,
+        0.13066797,
+        0.09283979,
+        0.1044982,
+        0.11446757,
+        0.12410894,
+        0.08575833,
+        0.08688664,
+        0.08871841,
+        0.09452496,
+        0.11280894,
+        0.08605019,
+        0.09069607,
+        0.09625262,
+        0.10480069,
+        0.08209087,
+        0.08907479,
+        0.09521613,
+        0.10375828,
+        0.0827678,
+        0.09389319,
+        0.09721766,
+        0.10260603,
+        0.0892784,
+        0.10309231,
+        0.11121955,
+        0.11911318,
+        0.08014706,
+        0.07939664,
+        0.07666104,
+        0.07640523,
+        0.07845239,
+        0.06779566,
+        0.06928173,
+        0.07995176,
+        0.09609538,
+        0.06776656,
+        0.07411631,
+        0.09502285,
+        0.11704809,
+        0.06976698,
+        0.07840788,
+        0.09568293,
+        0.11219386,
+        0.07334771,
+        0.07997227,
+        0.09204492,
+        0.10471888,
+        0.1324311,
+        0.13287905,
+        0.13296498,
+        0.13300247,
+        0.13251117,
+        0.13296743,
+        0.13352127,
+        0.13476767,
+        0.13467269,
+        0.13386367,
+        0.13339657,
+        0.13304512,
+        0.13318144,
+        0.13313657,
+        0.13394693,
+        0.13404495,
+        0.1343446,
+        0.13446471,
+        0.13349241,
+        0.13355125,
+        0.13414721,
+        0.13430822,
+        0.13283393,
+        0.13377732,
+        0.1346423,
+        0.13602652,
+        0.13584861,
+        0.13470158,
+        0.1339573,
+        0.13331288,
+        0.13342074,
+        0.133372,
+        0.13473015,
+        0.13483934,
+        0.13534908,
+        0.13551436,
+        0.13399816,
+        0.13405652,
+        0.1354323,
+        0.13537434,
+        0.06685787,
+        0.06737807,
+        0.06767439,
+        0.06927998,
+        0.06658512,
+        0.06643137,
+        0.0663855,
+        0.06645988,
+        0.06653237,
+        0.06679216,
+        0.06700299,
+        0.06721594,
+        0.06899743,
+        0.06748881,
+        0.06692849,
+        0.06752784,
+        0.06670087,
+        0.06690367,
+        0.06722134,
+        0.06834918,
+        0.06637124,
+        0.06663854,
+        0.06680202,
+        0.06691353,
+        0.06701645,
+        0.06724831,
+        0.06726662,
+        0.06730385,
+        0.06735906,
+        0.06739713,
+        0.06924284,
+        0.06767783,
+        0.06744281,
+        0.06815296,
+        0.06732813,
+        0.0676265,
+        0.06758311,
+        0.06880609,
+        0.06710069,
+        0.0672657
+    ]
+}

src/video_to_landmark_coordinates.py

@@ -0,0 +1,100 @@
+import cv2
+import mediapipe as mp
+import pandas as pd
+import numpy as np
+
+
+def generate_column_names():
+    """
+    Generate column names for a DataFrame that will store coordinates of landmarks.
+
+    Column names are formatted as '{coordinate}_{landmark_type}_{landmark_index}'.
+
+    Returns:
+    list: A list of strings representing the column names.
+    """
+    columns = ['frame']
+
+    # face columns
+    for coordinate in ['x', 'y']:
+        for i in range(468):  # Mediapipe face mesh contains 468 landmarks
+            columns.append(f'{coordinate}_face_{i}')
+
+    # hands columns
+    for hand in ['left_hand', 'right_hand']:
+        for coordinate in ['x', 'y']:
+            for i in range(21):  # Mediapipe hand model contains 21 landmarks
+                columns.append(f'{coordinate}_{hand}_{i}')
+
+    return columns
+
+
+def video_to_landmarks(video_path, columns):
+    """
+    Extract face and hand landmarks from a video and store them in a DataFrame.
+
+    The video is processed frame by frame. For each frame, face and hand landmarks
+    are detected using MediaPipe's face mesh and hand models, respectively.
+    The coordinates of the landmarks are stored in a DataFrame.
+
+    Parameters:
+    video_path (str): Path to the video file.
+    columns (list): List of column names for the DataFrame.
+
+    Returns:
+    pd.DataFrame: A DataFrame where each row corresponds to a frame and each column corresponds to a landmark.
+    """
+    mp_drawing = mp.solutions.drawing_utils
+    mp_face_mesh = mp.solutions.face_mesh
+    mp_hands = mp.solutions.hands
+
+    cap = cv2.VideoCapture(video_path)
+    df = pd.DataFrame(columns=columns)
+
+    with mp_face_mesh.FaceMesh() as face_mesh, mp_hands.Hands(max_num_hands=2) as hands:
+        frame_count = 0
+        while cap.isOpened():
+            success, frame = cap.read()
+            if not success:
+                break
+
+            rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results_face = face_mesh.process(rgb_frame)
+            results_hands = hands.process(rgb_frame)
+
+            # Initialize frame dictionary with NaNs
+            frame_data = {column: np.NaN for column in columns}
+            frame_data['frame'] = frame_count
+
+            # Process face landmarks
+            if results_face.multi_face_landmarks:
+                for face_landmarks in results_face.multi_face_landmarks:
+                    for i, landmark in enumerate(face_landmarks.landmark):
+                        frame_data[f'x_face_{i}'] = landmark.x
+                        frame_data[f'y_face_{i}'] = landmark.y
+
+            # Process hand landmarks
+            if results_hands.multi_hand_landmarks:
+                for hand_landmarks in results_hands.multi_hand_landmarks:
+                    if hand_landmarks.landmark[mp_hands.HandLandmark.WRIST].x < hand_landmarks.landmark[
+                        mp_hands.HandLandmark.THUMB_TIP].x:
+                        hand_type = 'left_hand'
+                    else:
+                        hand_type = 'right_hand'
+
+                    for i, landmark in enumerate(hand_landmarks.landmark):
+                        frame_data[f'x_{hand_type}_{i}'] = landmark.x
+                        frame_data[f'y_{hand_type}_{i}'] = landmark.y
+
+            df = df._append(frame_data, ignore_index=True)
+            frame_count += 1
+
+    cap.release()
+
+    return df
+
+# video_path = "videoplayback_with_landmarks.mp4"
+# df = video_to_landmarks(video_path, generate_column_names())
+#
+# # Save the DataFrame to a CSV file
+# df.to_csv('landmarks.csv', index=False)

src/video_with_landmarks.py

@@ -0,0 +1,89 @@
+import cv2
+import mediapipe as mp
+
+def process_video_with_landmarks(video_path, output_path, scale_percent=100):
+    """
+    Process a video to identify and draw landmarks on faces and hands.
+
+    Parameters:
+    video_path (str): The path to the input video file.
+    output_path (str): The path to the output video file.
+    scale_percent (int, optional): The percentage of the original size. Default is 100.
+    """
+    # MediaPipe solutions
+    mp_drawing = mp.solutions.drawing_utils
+    mp_face_mesh = mp.solutions.face_mesh
+    mp_hands = mp.solutions.hands
+
+    # Open the video file
+    cap = cv2.VideoCapture(video_path)
+
+    # Get the video properties
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = cap.get(cv2.CAP_PROP_FPS)
+
+    # Calculate the scale dimensions
+    width = int(width * scale_percent / 100)
+    height = int(height * scale_percent / 100)
+
+    # Define the output video file
+    # fourcc = cv2.VideoWriter_fourcc(*'h264')
+
+    fourcc = cv2.VideoWriter_fourcc(*'HEVC')
+    out_fps = fps / 0.6 # Set the output fps to half of the original fps
+    out = cv2.VideoWriter(output_path, fourcc, out_fps, (width, height))
+
+    # Process each frame
+    with mp_face_mesh.FaceMesh() as face_mesh, mp_hands.Hands() as hands:
+        while cap.isOpened():
+            success, frame = cap.read()
+            if not success:
+                break
+
+            # Resize the frame
+            frame = cv2.resize(frame, (width, height), interpolation = cv2.INTER_AREA)
+
+            # Convert the frame to RGB
+            rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+            # Process face landmarks
+            results_face = face_mesh.process(rgb_frame)
+            if results_face.multi_face_landmarks:
+                for face_landmarks in results_face.multi_face_landmarks:
+                    mp_drawing.draw_landmarks(
+                        frame,
+                        face_landmarks,
+                        mp_face_mesh.FACEMESH_TESSELATION,
+                        landmark_drawing_spec=mp_drawing.DrawingSpec(color=(255, 255, 255), thickness=1, circle_radius=1),
+                        connection_drawing_spec=mp_drawing.DrawingSpec(color=(255, 255, 255), thickness=1)
+                    )
+
+            # Process hand landmarks
+            results_hands = hands.process(rgb_frame)
+            if results_hands.multi_hand_landmarks:
+                for hand_landmarks in results_hands.multi_hand_landmarks:
+                    if hand_landmarks.landmark[mp_hands.HandLandmark.WRIST].x < hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP].x:
+                        landmark_color = (255, 0, 0)  # Left hand (Blue)
+                    else:
+                        landmark_color = (0, 0, 255)  # Right hand (Red)
+
+                    mp_drawing.draw_landmarks(
+                        frame,
+                        hand_landmarks,
+                        mp_hands.HAND_CONNECTIONS,
+                        landmark_drawing_spec=mp_drawing.DrawingSpec(color=landmark_color, thickness=1, circle_radius=1),
+                        connection_drawing_spec=mp_drawing.DrawingSpec(color=landmark_color, thickness=1)
+                    )
+
+            # Write the annotated frame to the output video
+            out.write(frame)
+
+            # If 'q' is pressed on the keyboard, exit this loop
+            if cv2.waitKey(1) & 0xFF == ord('q'):
+                break
+
+    # Close the video file
+    cap.release()
+    out.release()
+    cv2.destroyAllWindows()