Added other files

app.css

@@ -0,0 +1,100 @@
+div.app-flex-container {
+    display: flex;
+    align-items: left;
+}
+
+div.app-flex-container > a {
+    margin-left: 6px;
+}
+
+div.dl1 div.upload-container {
+    height: 350px;
+    max-height: 350px;
+}
+
+div.dl2 {
+    max-height: 200px;
+}
+
+div.dl2 img {
+    max-height: 200px;
+}
+
+div.dl5 {
+    max-height: 200px;
+}
+
+div.dl5 img {
+    max-height: 200px;
+}
+
+div.video1 div.video-container {
+    height: 500px;
+}
+
+div.video2 {
+    height: 200px;
+}
+
+div.video3 {
+    height: 200px;
+}
+
+div.video4 {
+    height: 200px;
+}
+
+div.stat {
+    height: 286px;
+}
+
+div.settings-wrapper {
+    display: none;
+}
+
+.submit {
+    display: inline-block;
+    padding: 10px 20px;
+    font-size: 16px;
+    font-weight: bold;
+    text-align: center;
+    text-decoration: none;
+    cursor: pointer;
+    border: var(--button-border-width) solid var(--button-primary-border-color);
+    background: var(--button-primary-background-fill);
+    color: var(--button-primary-text-color);
+    border-radius: 8px;
+    transition: all 0.3s ease;
+}
+
+.submit[disabled] {
+    cursor: not-allowed;
+    opacity: 0.6;
+}
+
+.submit:hover:not([disabled]) {
+    border-color: var(--button-primary-border-color-hover);
+    background: var(--button-primary-background-fill-hover);
+    color: var(--button-primary-text-color-hover);
+}
+
+.clear {
+    display: inline-block;
+    padding: 10px 20px;
+    font-size: 16px;
+    font-weight: bold;
+    text-align: center;
+    text-decoration: none;
+    cursor: pointer;
+    border-radius: 8px;
+    transition: all 0.3s ease;
+}
+
+.clear[disabled] {
+    cursor: not-allowed;
+    opacity: 0.6;
+}
+
+.submit:active:not([disabled]), .clear:active:not([disabled]) {
+    transform: scale(0.98);
+}

app.py

@@ -1,7 +1,131 @@
+"""
+File: app.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Description: Main application file for Facial_Expression_Recognition.
+             The file defines the Gradio interface, sets up the main blocks,
+             and includes event handlers for various components.
+License: MIT License
+"""
+
 import gradio as gr
 
-def greet(name):
-    return "Hello " + name + "!!"
+# Importing necessary components for the Gradio app
+from app.description import DESCRIPTION_STATIC, DESCRIPTION_DYNAMIC
+from app.authors import AUTHORS
+from app.app_utils import preprocess_image_and_predict, preprocess_video_and_predict
+
+
+def clear_static_info():
+    return (
+        gr.Image(value=None, type="pil"),
+        gr.Image(value=None, scale=1, elem_classes="dl5"),
+        gr.Image(value=None, scale=1, elem_classes="dl2"),
+        gr.Label(value=None, num_top_classes=3, scale=1, elem_classes="dl3"),
+    )
+
+def clear_dynamic_info():
+    return (
+        gr.Video(value=None),
+        gr.Video(value=None),
+        gr.Video(value=None),
+        gr.Video(value=None),
+        gr.Plot(value=None),
+    )
+
+with gr.Blocks(css="app.css") as demo:
+    with gr.Tab("Dynamic App"):
+        gr.Markdown(value=DESCRIPTION_DYNAMIC)
+        with gr.Row():
+            with gr.Column(scale=2):
+                input_video = gr.Video(elem_classes="video1")
+                with gr.Row():
+                    clear_btn_dynamic = gr.Button(
+                        value="Clear", interactive=True, scale=1
+                    )
+                    submit_dynamic = gr.Button(
+                        value="Submit", interactive=True, scale=1, elem_classes="submit"
+                    )
+            with gr.Column(scale=2, elem_classes="dl4"):
+                with gr.Row():
+                    output_video = gr.Video(label="Original video", scale=1, elem_classes="video2")
+                    output_face = gr.Video(label="Pre-processed video", scale=1, elem_classes="video3")
+                    output_heatmaps = gr.Video(label="Heatmaps", scale=1, elem_classes="video4")
+                output_statistics = gr.Plot(label="Statistics of emotions", elem_classes="stat")
+        gr.Examples(
+            ["videos/video1.mp4",
+            "videos/video2.mp4",
+            ],
+            [input_video],
+        )
+
+    with gr.Tab("Static App"):
+        gr.Markdown(value=DESCRIPTION_STATIC)
+        with gr.Row():
+            with gr.Column(scale=2, elem_classes="dl1"):
+                input_image = gr.Image(label="Original image", type="pil")
+                with gr.Row():
+                    clear_btn = gr.Button(
+                        value="Clear", interactive=True, scale=1, elem_classes="clear"
+                    )
+                    submit = gr.Button(
+                        value="Submit", interactive=True, scale=1, elem_classes="submit"
+                    )
+            with gr.Column(scale=1, elem_classes="dl4"):
+                with gr.Row():
+                    output_image = gr.Image(label="Face", scale=1, elem_classes="dl5")
+                    output_heatmap = gr.Image(label="Heatmap", scale=1, elem_classes="dl2")
+                output_label = gr.Label(num_top_classes=3, scale=1, elem_classes="dl3")
+        gr.Examples(
+            [
+                "images/fig7.jpg",
+                "images/fig1.jpg",
+                "images/fig2.jpg",
+                "images/fig3.jpg",
+                "images/fig4.jpg",
+                "images/fig5.jpg",
+                "images/fig6.jpg",
+            ],
+            [input_image],
+        )
+    with gr.Tab("Authors"):
+        gr.Markdown(value=AUTHORS)
+
+    submit.click(
+        fn=preprocess_image_and_predict,
+        inputs=[input_image],
+        outputs=[output_image, output_heatmap, output_label],
+        queue=True,
+    )
+    clear_btn.click(
+        fn=clear_static_info,
+        inputs=[],
+        outputs=[input_image, output_image, output_heatmap, output_label],
+        queue=True,
+    )
+
+    submit_dynamic.click(
+        fn=preprocess_video_and_predict,
+        inputs=input_video,
+        outputs=[
+            output_video,
+            output_face,
+            output_heatmaps, 
+            output_statistics
+        ],
+        queue=True,
+    )
+    clear_btn_dynamic.click(
+        fn=clear_dynamic_info,
+        inputs=[],
+        outputs=[
+            input_video,
+            output_video,
+            output_face,
+            output_heatmaps, 
+            output_statistics
+        ],
+        queue=True,
+    )
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+if __name__ == "__main__":
+    demo.queue(api_open=False).launch(share=False)

app/app_utils.py

@@ -0,0 +1,174 @@
+"""
+File: app_utils.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module contains utility functions for facial expression recognition application.
+License: MIT License
+"""
+
+import torch
+import numpy as np
+import mediapipe as mp
+from PIL import Image
+import cv2
+from pytorch_grad_cam.utils.image import show_cam_on_image
+
+# Importing necessary components for the Gradio app
+from app.model import pth_model_static, pth_model_dynamic, cam, pth_processing
+from app.face_utils import get_box, display_info
+from app.config import DICT_EMO, config_data
+from app.plot import statistics_plot
+
+mp_face_mesh = mp.solutions.face_mesh
+
+
+def preprocess_image_and_predict(inp):
+    inp = np.array(inp)
+
+    if inp is None:
+        return None, None, None
+
+    try:
+        h, w = inp.shape[:2]
+    except Exception:
+        return None, None, None
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5,
+    ) as face_mesh:
+        results = face_mesh.process(inp)
+        if results.multi_face_landmarks:
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = inp[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                with torch.no_grad():
+                    prediction = (
+                        torch.nn.functional.softmax(pth_model_static(cur_face_n), dim=1)
+                        .detach()
+                        .numpy()[0]
+                    )
+                confidences = {DICT_EMO[i]: float(prediction[i]) for i in range(7)}
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face,(224,224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+
+            return cur_face, heatmap, confidences
+        
+        else:
+            return None, None, None
+
+def preprocess_video_and_predict(video):
+
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_face = 'result_face.mp4'
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    path_save_video_hm = 'result_hm.mp4'
+    vid_writer_hm = cv2.VideoWriter(path_save_video_hm, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    lstm_features = []
+    count_frame = 1
+    count_face = 0
+    probs = []
+    frames = []
+    last_output = None
+    last_heatmap = None 
+    cur_face = None
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            _, frame = cap.read()
+            if frame is None: break
+
+            frame_copy = frame.copy()
+            frame_copy.flags.writeable = False
+            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_copy)
+            frame_copy.flags.writeable = True
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY  = get_box(fl, w, h)
+                    cur_face = frame_copy[startY:endY, startX: endX]
+
+                    if count_face%config_data.FRAME_DOWNSAMPLING == 0:
+                        cur_face_copy = pth_processing(Image.fromarray(cur_face))
+                        with torch.no_grad():
+                            features = torch.nn.functional.relu(pth_model_static.extract_features(cur_face_copy)).detach().numpy()
+
+                        grayscale_cam = cam(input_tensor=cur_face_copy)
+                        grayscale_cam = grayscale_cam[0, :]
+                        cur_face_hm = cv2.resize(cur_face,(224,224), interpolation = cv2.INTER_AREA)
+                        cur_face_hm = np.float32(cur_face_hm) / 255
+                        heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=False)
+                        last_heatmap = heatmap
+        
+                        if len(lstm_features) == 0:
+                            lstm_features = [features]*10
+                        else:
+                            lstm_features = lstm_features[1:] + [features]
+
+                        lstm_f = torch.from_numpy(np.vstack(lstm_features))
+                        lstm_f = torch.unsqueeze(lstm_f, 0)
+                        with torch.no_grad():
+                            output = pth_model_dynamic(lstm_f).detach().numpy()
+                        last_output = output
+
+                        if count_face == 0:
+                            count_face += 1
+
+                    else:
+                        if last_output is not None:
+                            output = last_output
+                            heatmap = last_heatmap
+
+                        elif last_output is None:
+                            output = np.empty((1, 7))
+                            output[:] = np.nan
+                            
+                    probs.append(output[0])
+                    frames.append(count_frame)
+            else:
+                if last_output is not None:
+                    lstm_features = []
+                    empty = np.empty((7))
+                    empty[:] = np.nan
+                    probs.append(empty)
+                    frames.append(count_frame)                        
+
+            if cur_face is not None:
+                heatmap_f = display_info(heatmap, 'Frame: {}'.format(count_frame), box_scale=.3)
+
+                cur_face = cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR)
+                cur_face = cv2.resize(cur_face, (224,224), interpolation = cv2.INTER_AREA)
+                cur_face = display_info(cur_face, 'Frame: {}'.format(count_frame), box_scale=.3)
+                vid_writer_face.write(cur_face)
+                vid_writer_hm.write(heatmap_f)
+
+            count_frame += 1
+            if count_face != 0:
+                count_face += 1
+
+        vid_writer_face.release()
+        vid_writer_hm.release()
+
+        stat = statistics_plot(frames, probs)
+
+        if not stat:
+            return None, None, None, None
+        
+    return video, path_save_video_face, path_save_video_hm, stat

app/authors.py

@@ -0,0 +1,12 @@
+"""
+File: authors.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: About the authors.
+License: MIT License
+"""
+
+
+AUTHORS = """
+    Authors: [Elena Ryumina](https://github.com/ElenaRyumina), [Dmitry Ryumin](https://github.com/DmitryRyumin), [Denis Dresvyanskiy](https://www.uni-ulm.de/en/nt/staff/research-assistants/dresvyanskiy/), [Maxim Markitantov](https://hci.nw.ru/en/employees/10) and [Alexey Karpov](https://hci.nw.ru/en/employees/1)
+
+"""

app/config.py

@@ -0,0 +1,49 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Configuration file.
+License: MIT License
+"""
+
+import toml
+from typing import Dict
+from types import SimpleNamespace
+
+
+def flatten_dict(prefix: str, d: Dict) -> Dict:
+    result = {}
+
+    for k, v in d.items():
+        if isinstance(v, dict):
+            result.update(flatten_dict(f"{prefix}{k}_", v))
+        else:
+            result[f"{prefix}{k}"] = v
+
+    return result
+
+
+config = toml.load("config.toml")
+
+config_data = flatten_dict("", config)
+
+config_data = SimpleNamespace(**config_data)
+
+DICT_EMO = {
+    0: "Neutral",
+    1: "Happiness",
+    2: "Sadness",
+    3: "Surprise",
+    4: "Fear",
+    5: "Disgust",
+    6: "Anger",
+}
+
+COLORS = {
+    0: 'blue', 
+    1: 'orange', 
+    2: 'green', 
+    3: 'red', 
+    4: 'purple', 
+    5: 'brown', 
+    6: 'pink'
+}

app/description.py

@@ -0,0 +1,19 @@
+"""
+File: description.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Project description for the Gradio app.
+License: MIT License
+"""
+
+# Importing necessary components for the Gradio app
+from app.config import config_data
+
+DESCRIPTION_STATIC = f"""\
+# Static Facial Expression Recognition
+
+"""
+
+DESCRIPTION_DYNAMIC = f"""\
+# Dynamic Facial Expression Recognition
+
+"""

app/face_utils.py

@@ -0,0 +1,68 @@
+"""
+File: face_utils.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module contains utility functions related to facial landmarks and image processing.
+License: MIT License
+"""
+
+import numpy as np
+import math
+import cv2
+
+
+def norm_coordinates(normalized_x, normalized_y, image_width, image_height):
+    x_px = min(math.floor(normalized_x * image_width), image_width - 1)
+    y_px = min(math.floor(normalized_y * image_height), image_height - 1)
+    return x_px, y_px
+
+
+def get_box(fl, w, h):
+    idx_to_coors = {}
+    for idx, landmark in enumerate(fl.landmark):
+        landmark_px = norm_coordinates(landmark.x, landmark.y, w, h)
+        if landmark_px:
+            idx_to_coors[idx] = landmark_px
+
+    x_min = np.min(np.asarray(list(idx_to_coors.values()))[:, 0])
+    y_min = np.min(np.asarray(list(idx_to_coors.values()))[:, 1])
+    endX = np.max(np.asarray(list(idx_to_coors.values()))[:, 0])
+    endY = np.max(np.asarray(list(idx_to_coors.values()))[:, 1])
+
+    (startX, startY) = (max(0, x_min), max(0, y_min))
+    (endX, endY) = (min(w - 1, endX), min(h - 1, endY))
+
+    return startX, startY, endX, endY
+
+def display_info(img, text, margin=1.0, box_scale=1.0):
+    img_copy = img.copy()
+    img_h, img_w, _ = img_copy.shape
+    line_width = int(min(img_h, img_w) * 0.001)
+    thickness = max(int(line_width / 3), 1)
+
+    font_face = cv2.FONT_HERSHEY_SIMPLEX
+    font_color = (0, 0, 0)
+    font_scale = thickness / 1.5
+
+    t_w, t_h = cv2.getTextSize(text, font_face, font_scale, None)[0]
+
+    margin_n = int(t_h * margin)
+    sub_img = img_copy[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+              img_w - t_w - margin_n - int(2 * t_h * box_scale): img_w - margin_n]
+
+    white_rect = np.ones(sub_img.shape, dtype=np.uint8) * 255
+
+    img_copy[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+    img_w - t_w - margin_n - int(2 * t_h * box_scale):img_w - margin_n] = cv2.addWeighted(sub_img, 0.5, white_rect, .5, 1.0)
+
+    cv2.putText(img=img_copy,
+                text=text,
+                org=(img_w - t_w - margin_n - int(2 * t_h * box_scale) // 2,
+                     0 + margin_n + t_h + int(2 * t_h * box_scale) // 2),
+                fontFace=font_face,
+                fontScale=font_scale,
+                color=font_color,
+                thickness=thickness,
+                lineType=cv2.LINE_AA,
+                bottomLeftOrigin=False)
+
+    return img_copy

app/model.py

@@ -0,0 +1,64 @@
+"""
+File: model.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module provides functions for loading and processing a pre-trained deep learning model
+             for facial expression recognition.
+License: MIT License
+"""
+
+import torch
+import requests
+from PIL import Image
+from torchvision import transforms
+from pytorch_grad_cam import GradCAM
+
+# Importing necessary components for the Gradio app
+from app.config import config_data
+from app.model_architectures import ResNet50, LSTMPyTorch
+
+
+def load_model(model_url, model_path):
+    try:
+        with requests.get(model_url, stream=True) as response:
+            with open(model_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+        return model_path
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        return None
+
+path_static = load_model(config_data.model_static_url, config_data.model_static_path)   
+pth_model_static = ResNet50(7, channels=3)
+pth_model_static.load_state_dict(torch.load(path_static))
+pth_model_static.eval()
+
+path_dynamic = load_model(config_data.model_dynamic_url, config_data.model_dynamic_path) 
+pth_model_dynamic = LSTMPyTorch()
+pth_model_dynamic.load_state_dict(torch.load(path_dynamic))
+pth_model_dynamic.eval()
+
+target_layers = [pth_model_static.layer4]
+cam = GradCAM(model=pth_model_static, target_layers=target_layers)
+
+def pth_processing(fp):
+    class PreprocessInput(torch.nn.Module):
+        def init(self):
+            super(PreprocessInput, self).init()
+
+        def forward(self, x):
+            x = x.to(torch.float32)
+            x = torch.flip(x, dims=(0,))
+            x[0, :, :] -= 91.4953
+            x[1, :, :] -= 103.8827
+            x[2, :, :] -= 131.0912
+            return x
+
+    def get_img_torch(img, target_size=(224, 224)):
+        transform = transforms.Compose([transforms.PILToTensor(), PreprocessInput()])
+        img = img.resize(target_size, Image.Resampling.NEAREST)
+        img = transform(img)
+        img = torch.unsqueeze(img, 0)
+        return img
+
+    return get_img_torch(fp)

app/model_architectures.py

@@ -0,0 +1,150 @@
+"""
+File: model.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module provides model architectures.
+License: MIT License
+"""
+
+import torch
+import torch.nn as  nn
+import torch.nn.functional as F
+import math
+
+class Bottleneck(nn.Module):
+    expansion = 4
+    def __init__(self, in_channels, out_channels, i_downsample=None, stride=1):
+        super(Bottleneck, self).__init__()
+        
+        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, padding=0, bias=False)
+        self.batch_norm1 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)
+        
+        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding='same', bias=False)
+        self.batch_norm2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)
+        
+        self.conv3 = nn.Conv2d(out_channels, out_channels*self.expansion, kernel_size=1, stride=1, padding=0, bias=False)
+        self.batch_norm3 = nn.BatchNorm2d(out_channels*self.expansion, eps=0.001, momentum=0.99)
+        
+        self.i_downsample = i_downsample
+        self.stride = stride
+        self.relu = nn.ReLU()
+        
+    def forward(self, x):
+        identity = x.clone()
+        x = self.relu(self.batch_norm1(self.conv1(x)))
+        
+        x = self.relu(self.batch_norm2(self.conv2(x)))
+        
+        x = self.conv3(x)
+        x = self.batch_norm3(x)
+        
+        #downsample if needed
+        if self.i_downsample is not None:
+            identity = self.i_downsample(identity)
+        #add identity
+        x+=identity
+        x=self.relu(x)
+        
+        return x
+
+class Conv2dSame(torch.nn.Conv2d):
+
+    def calc_same_pad(self, i: int, k: int, s: int, d: int) -> int:
+        return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        ih, iw = x.size()[-2:]
+
+        pad_h = self.calc_same_pad(i=ih, k=self.kernel_size[0], s=self.stride[0], d=self.dilation[0])
+        pad_w = self.calc_same_pad(i=iw, k=self.kernel_size[1], s=self.stride[1], d=self.dilation[1])
+
+        if pad_h > 0 or pad_w > 0:
+            x = F.pad(
+                x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]
+            )
+        return F.conv2d(
+            x,
+            self.weight,
+            self.bias,
+            self.stride,
+            self.padding,
+            self.dilation,
+            self.groups,
+        )
+
+class ResNet(nn.Module):
+    def __init__(self, ResBlock, layer_list, num_classes, num_channels=3):
+        super(ResNet, self).__init__()
+        self.in_channels = 64
+
+        self.conv_layer_s2_same = Conv2dSame(num_channels, 64, 7, stride=2, groups=1, bias=False)
+        self.batch_norm1 = nn.BatchNorm2d(64, eps=0.001, momentum=0.99)
+        self.relu = nn.ReLU()
+        self.max_pool = nn.MaxPool2d(kernel_size = 3, stride=2)
+        
+        self.layer1 = self._make_layer(ResBlock, layer_list[0], planes=64, stride=1)
+        self.layer2 = self._make_layer(ResBlock, layer_list[1], planes=128, stride=2)
+        self.layer3 = self._make_layer(ResBlock, layer_list[2], planes=256, stride=2)
+        self.layer4 = self._make_layer(ResBlock, layer_list[3], planes=512, stride=2)
+        
+        self.avgpool = nn.AdaptiveAvgPool2d((1,1))
+        self.fc1 = nn.Linear(512*ResBlock.expansion, 512)
+        self.relu1 = nn.ReLU()
+        self.fc2 = nn.Linear(512, num_classes)
+
+    def extract_features(self, x):
+        x = self.relu(self.batch_norm1(self.conv_layer_s2_same(x)))
+        x = self.max_pool(x)
+        # print(x.shape)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        
+        x = self.avgpool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+        return x
+        
+    def forward(self, x):
+        x = self.extract_features(x)
+        x = self.relu1(x)
+        x = self.fc2(x)
+        return x
+        
+    def _make_layer(self, ResBlock, blocks, planes, stride=1):
+        ii_downsample = None
+        layers = []
+        
+        if stride != 1 or self.in_channels != planes*ResBlock.expansion:
+            ii_downsample = nn.Sequential(
+                nn.Conv2d(self.in_channels, planes*ResBlock.expansion, kernel_size=1, stride=stride, bias=False, padding=0),
+                nn.BatchNorm2d(planes*ResBlock.expansion, eps=0.001, momentum=0.99)
+            )
+            
+        layers.append(ResBlock(self.in_channels, planes, i_downsample=ii_downsample, stride=stride))
+        self.in_channels = planes*ResBlock.expansion
+        
+        for i in range(blocks-1):
+            layers.append(ResBlock(self.in_channels, planes))
+            
+        return nn.Sequential(*layers)
+        
+def ResNet50(num_classes, channels=3):
+    return ResNet(Bottleneck, [3,4,6,3], num_classes, channels)
+
+
+class LSTMPyTorch(nn.Module):
+    def __init__(self):
+        super(LSTMPyTorch, self).__init__()
+        
+        self.lstm1 = nn.LSTM(input_size=512, hidden_size=512, batch_first=True, bidirectional=False)
+        self.lstm2 = nn.LSTM(input_size=512, hidden_size=256, batch_first=True, bidirectional=False)
+        self.fc = nn.Linear(256, 7)
+        self.softmax = nn.Softmax(dim=1)
+
+    def forward(self, x):
+        x, _ = self.lstm1(x)
+        x, _ = self.lstm2(x)        
+        x = self.fc(x[:, -1, :])
+        x = self.softmax(x)
+        return x

app/plot.py

@@ -0,0 +1,29 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Plotting statistical information.
+License: MIT License
+"""
+import matplotlib.pyplot as plt
+import numpy as np 
+
+# Importing necessary components for the Gradio app
+from app.config import DICT_EMO, COLORS
+
+
+def statistics_plot(frames, probs):
+    fig, ax = plt.subplots(figsize=(10, 4))
+    fig.subplots_adjust(left=0.07, bottom=0.14, right=0.98, top=0.8, wspace=0, hspace=0)
+    # Установка параметров left, bottom, right, top, чтобы выделить место для легенды и названий осей
+    probs = np.array(probs)
+    for i in range(7):
+        try:
+            ax.plot(frames, probs[:, i], label=DICT_EMO[i], c=COLORS[i]) 
+        except Exception:
+            return None
+            
+    ax.legend(loc='upper center', bbox_to_anchor=(0.47, 1.2), ncol=7, fontsize=12) 
+    ax.set_xlabel('Frames', fontsize=12)  # Добавляем подпись к оси X
+    ax.set_ylabel('Probability', fontsize=12)  # Добавляем подпись к оси Y
+    ax.grid(True)
+    return plt

config.toml

@@ -0,0 +1,10 @@
+APP_VERSION = "0.2.0"
+FRAME_DOWNSAMPLING = 5
+
+[model_static]
+url = "https://huggingface.co/ElenaRyumina/face_emotion_recognition/resolve/main/FER_static_ResNet50_AffectNet.pt"
+path = "FER_static_ResNet50_AffectNet.pt"
+
+[model_dynamic]
+url = "https://huggingface.co/ElenaRyumina/face_emotion_recognition/resolve/main/FER_dinamic_LSTM_IEMOCAP.pt"
+path = "FER_dinamic_LSTM_IEMOCAP.pt"

requirements.txt

@@ -0,0 +1,9 @@
+gradio>=4.24.0
+requests>=2.31.0
+torch>=2.1.2
+torchaudio>=2.1.2
+torchvision>=0.16.2
+mediapipe>=0.10.9
+pillow>=10.2.0
+toml>=0.10.2
+grad-cam>=1.5.0