Upload 5 files

__init__.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# MIT License
+#
+# Copyright (c) 2018 Justin Shenk
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import logging
+
+from .classes import Video
+from .fer import FER
+
+log = logging.getLogger("fer")
+log.setLevel(logging.INFO)
+
+__version__ = "22.5.1"
+
+__title__ = "fer"
+__description__ = "Facial expression recognition from images"
+__url__ = "https://github.com/justinshenk/fer"
+__uri__ = __url__
+__doc__ = __description__ + " <" + __url__ + ">"
+
+__author__ = "Justin Shenk"
+__email__ = "[email protected]"
+
+__license__ = "MIT"
+__copyright__ = "Copyright (c) 2019 " + __author__

classes.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import csv
+import logging
+import os
+import re
+from moviepy import VideoFileClip,AudioFileClip, CompositeAudioClip
+from pathlib import Path
+from typing import Optional, Union
+from zipfile import ZipFile
+
+import cv2
+import pandas as pd
+
+from tqdm import tqdm
+from tqdm.contrib.logging import logging_redirect_tqdm
+
+from .utils import draw_annotations
+
+log = logging.getLogger("fer")
+
+
+class Video(object):
+    def __init__(
+        self,
+        video_file: str,
+        outdir: str = "output",
+        first_face_only: bool = True,
+        tempfile: Optional[str] = None,
+    ):
+        """Video class for extracting and saving frames for emotion detection.
+        :param video_file - str
+        :param outdir - str
+        :param tempdir - str
+        :param first_face_only - bool
+        :param tempfile - str
+        """
+        assert os.path.exists(video_file), "Video file not found at {}".format(
+            os.path.abspath(video_file)
+        )
+        self.cap = cv2.VideoCapture(video_file)
+        if not os.path.isdir(outdir):
+            os.makedirs(outdir, exist_ok=True)
+        self.outdir = outdir
+
+        if not first_face_only:
+            log.error("Only single-face charting is implemented")
+        self.first_face_only = first_face_only
+        self.tempfile = tempfile
+        self.filepath = video_file
+        self.filename = "".join(self.filepath.split("/")[-1])
+
+    @staticmethod
+    def get_max_faces(data: list) -> int:
+        """Get max number of faces detected in a series of frames, eg 3"""
+        max = 0
+        for frame in data:
+            for face in frame:
+                if len(face) > max:
+                    max = len(face)
+        return max
+
+    @staticmethod
+    def _to_dict(data: Union[dict, list]) -> dict:
+        emotions = []
+
+        frame = data[0]
+        if isinstance(frame, list):
+            try:
+                emotions = frame[0]["emotions"].keys()
+            except IndexError:
+                raise Exception("No data in 'data'")
+        elif isinstance(frame, dict):
+            return data
+
+        dictlist = []
+
+        for data_idx, frame in enumerate(data):
+            rowdict = {}
+            for idx, face in enumerate(list(frame)):
+                if not isinstance(face, dict):
+                    break
+                rowdict.update({"box" + str(idx): face["box"]})
+                rowdict.update(
+                    {emo + str(idx): face["emotions"][emo] for emo in emotions}
+                )
+            dictlist.append(rowdict)
+        return dictlist
+
+    def to_pandas(self, data: Union[pd.DataFrame, list]) -> pd.DataFrame:
+        """Convert results to pandas DataFrame"""
+        if isinstance(data, pd.DataFrame):
+            return data
+
+        if not len(data):
+            return pd.DataFrame()
+        datalist = self._to_dict(data)
+        df = pd.DataFrame(datalist)
+        if self.first_face_only:
+            df = self.get_first_face(df)
+        return df
+
+    @staticmethod
+    def get_first_face(df: pd.DataFrame) -> pd.DataFrame:
+        assert isinstance(df, pd.DataFrame), "Must be a pandas DataFrame"
+        try:
+            int(df.columns[0][-1])
+        except ValueError:
+            # Already only one face in df
+            return df
+
+        columns = [x for x in df.columns if x[-1] == "0"]
+        new_columns = [x[:-1] for x in columns]
+        single_df = df[columns]
+        single_df.columns = new_columns
+        return single_df
+
+    @staticmethod
+    def get_emotions(df: pd.DataFrame) -> list:
+        """Get emotion columsn from results."""
+        columns = [x for x in df.columns if "box" not in x]
+        return df[columns]
+
+    def to_csv(self, data, filename="data.csv"):
+        """Save data to csv"""
+
+        def key(item):
+            key_pat = re.compile(r"^(\D+)(\d+)$")
+            m = key_pat.match(item)
+            return m.group(1), int(m.group(2))
+
+        dictlist = self._to_dict(data)
+        columns = set().union(*(d.keys() for d in dictlist))
+        columns = sorted(columns, key=key)  # sort by trailing number (faces)
+
+        with open("data.csv", "w", newline="") as csvfile:
+            writer = csv.DictWriter(csvfile, columns, lineterminator="\n")
+            writer.writeheader()
+            writer.writerows(dictlist)
+        return dictlist
+
+    def _close_video(self, outfile, save_frames, zip_images):
+        self.cap.release()
+        if self.display or self.save_video:
+            self.videowriter.release()
+
+        if self.save_video:
+            log.info("Completed analysis: saved to {}".format(self.tempfile or outfile))
+            if self.tempfile:
+                os.replace(self.tempfile, outfile)
+
+        if save_frames and zip_images:
+            log.info("Starting to Zip")
+            outdir = Path(self.outdir)
+            zip_dir = outdir / "images.zip"
+            images = sorted(list(outdir.glob("*.jpg")))
+            total = len(images)
+            i = 0
+            with ZipFile(zip_dir, "w") as zip:
+                for file in images:
+                    zip.write(file, arcname=file.name)
+                    os.remove(file)
+                    i += 1
+                    if i % 50 == 0:
+                        log.info(f"Compressing: {i*100 // total}%")
+            log.info("Zip has finished")
+
+    def _offset_detection_box(self, faces, detection_box):
+        for face in faces:
+            original_box = face.get("box")
+            face["box"] = (
+                original_box[0] + detection_box.get("x_min"),
+                original_box[1] + detection_box.get("y_min"),
+                original_box[2],
+                original_box[3],
+            )
+        return faces
+
+    def _increment_frames(
+        self, frame, faces, video_id, root, lang="en", size_multiplier=1
+    ):
+        # Save images to `self.outdir`
+        imgpath = os.path.join(
+            self.outdir, (video_id or root) + str(self.frameCount) + ".jpg"
+        )
+
+        if self.annotate_frames:
+            frame = draw_annotations(
+                frame,
+                faces,
+                boxes=True,
+                scores=True,
+                lang=lang,
+                size_multiplier=size_multiplier,
+            )
+
+        if self.save_frames:
+            cv2.imwrite(imgpath, frame)
+
+        if self.display:
+            cv2.imshow("Video", frame)
+
+        if self.save_video:
+            self.videowriter.write(frame)
+
+        self.frameCount += 1
+
+    def analyze(
+        self,
+        detector,  # fer.FER instance
+        display: bool = False,
+        output: str = "csv",
+        frequency: Optional[int] = None,
+        max_results: int = None,
+        save_fps: Optional[int] = None,
+        video_id: Optional[str] = None,
+        save_frames: bool = True,
+        save_video: bool = True,
+        annotate_frames: bool = True,
+        zip_images: bool = True,
+        detection_box: Optional[dict] = None,
+        lang: str = "en",
+        include_audio: bool = False,
+        size_multiplier: int = 1,
+    ) -> list:
+        """Recognize facial expressions in video using `detector`.
+
+        Args:
+
+            detector (fer.FER): facial expression recognizer
+            display (bool): show images with cv2.imshow
+            output (str): csv or pandas
+            frequency (int): inference on every nth frame (higher number is faster)
+            max_results (int): number of frames to run inference before stopping
+            save_fps (bool): inference frequency = video fps // save_fps
+            video_id (str): filename for saving
+            save_frames (bool): saves frames to directory
+            save_video (bool): saves output video
+            annotate_frames (bool): add emotion labels
+            zip_images (bool): compress output
+            detection_box (dict): dict with bounding box for subimage (xmin, xmax, ymin, ymax)
+            lang (str): emotion language that will be shown on video
+            include_audio (bool): indicates if a sounded version of the prediction video should be created or not
+            size_multiplier (int): increases the size of emotion labels shown in the video by x(size_multiplier)
+        Returns:
+
+            data (list): list of results
+
+        """
+        frames_emotions = []
+        if frequency is None:
+            frequency = 1
+        else:
+            frequency = int(frequency)
+
+        self.display = display
+        self.save_frames = save_frames
+        self.save_video = save_video
+        self.annotate_frames = annotate_frames
+
+        results_nr = 0
+
+        # Open video
+        assert self.cap.open(self.filepath), "Video capture not opening"
+        self.__emotions = detector._get_labels().items()
+        self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
+        pos_frames = self.cap.get(cv2.CAP_PROP_POS_FRAMES)
+        assert int(pos_frames) == 0, "Video not at index 0"
+
+        self.frameCount = 0
+        height, width = (
+            int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
+            int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
+        )
+
+        fps = self.cap.get(cv2.CAP_PROP_FPS)
+        length = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        assert fps and length, "File {} not loaded".format(self.filepath)
+
+        if save_fps is not None:
+            frequency = fps // save_fps
+            log.info("Saving every {} frames".format(frequency))
+
+        log.info(
+            "{:.2f} fps, {} frames, {:.2f} seconds".format(fps, length, length / fps)
+        )
+
+        if self.save_frames:
+            os.makedirs(self.outdir, exist_ok=True)
+            log.info(f"Making directories at {self.outdir}")
+        root, ext = os.path.splitext(os.path.basename(self.filepath))
+        outfile = os.path.join(self.outdir, f"{root}_output{ext}")
+
+        if save_video:
+            self.videowriter = self._save_video(outfile, fps, width, height)
+
+        with logging_redirect_tqdm():
+            pbar = tqdm(total=length, unit="frames")
+
+        while self.cap.isOpened():
+            ret, frame = self.cap.read()
+            if not ret:  # end of video
+                break
+
+            if frame is None:
+                log.warn("Empty frame")
+                continue
+
+            if self.frameCount % frequency != 0:
+                self.frameCount += 1
+                continue
+
+            if detection_box is not None:
+                frame = self._crop(frame, detection_box)
+
+            # Get faces and detect emotions; coordinates are for unpadded frame
+            try:
+                faces = detector.detect_emotions(frame)
+            except Exception as e:
+                log.error(e)
+                break
+
+            # Offset detection_box to include padding
+            if detection_box is not None:
+                faces = self._offset_detection_box(faces, detection_box)
+
+            self._increment_frames(frame, faces, video_id, root, lang, size_multiplier)
+
+            if cv2.waitKey(1) & 0xFF == ord("q"):
+                break
+
+            if faces:
+                frames_emotions.append(faces)
+
+            results_nr += 1
+            if max_results and results_nr > max_results:
+                break
+
+            pbar.update(1)
+
+        pbar.close()
+        self._close_video(outfile, save_frames, zip_images)
+
+        if include_audio:
+            audio_suffix = "_audio."
+            my_audio = AudioFileClip(self.filepath)
+            new_audioclip = CompositeAudioClip([my_audio])
+
+            my_output_clip = VideoFileClip(outfile)
+            my_output_clip.audio = new_audioclip
+            my_output_clip.write_videofile(audio_suffix.join(outfile.rsplit(".", 1)))
+
+        return self.to_format(frames_emotions, output)
+
+    def to_format(self, data, format):
+        """Return data in format."""
+        methods_lookup = {"csv": self.to_csv, "pandas": self.to_pandas}
+        return methods_lookup[format](data)
+
+    def _save_video(self, outfile: str, fps: int, width: int, height: int):
+        if os.path.isfile(outfile):
+            os.remove(outfile)
+            log.info("Deleted pre-existing {}".format(outfile))
+        if self.tempfile and os.path.isfile(self.tempfile):
+            os.remove(self.tempfile)
+        fourcc = cv2.VideoWriter_fourcc("m", "p", "4", "v")
+        videowriter = cv2.VideoWriter(
+            self.tempfile or outfile, fourcc, fps, (width, height), True
+        )
+        return videowriter
+
+    @staticmethod
+    def _crop(frame, detection_box):
+        crop_frame = frame[
+            detection_box.get("y_min") : detection_box.get("y_max"),
+            detection_box.get("x_min") : detection_box.get("x_max"),
+        ]
+        return crop_frame
+
+    def __del__(self):
+        cv2.destroyAllWindows()

emotionsmultilanguage.py

@@ -0,0 +1,20 @@
+"""
+When you add new language translation, you need to add the translations for each key element (angry, disgust, fear, happy, sad, suprise, netural)
+with the corresponding language key. Please be careful about the English characters. I.e. Wutend is originally Wütend but since 'ü' is not in 
+en alphabet we should change it to 'u'
+
+Languages Added:
+"en": English -- It's default language and no need to be added again. Program will read en values from keys of this dictionary
+"tr": Turkish (Türkçe)
+"de": German (Deutsch)
+"""
+
+emotions_dict = {
+    "angry": {"tr": "Kizgin", "de": "Wutend"},
+    "disgust": {"tr": "Igrenme", "de": "der Ekel"},
+    "fear": {"tr": "Korku", "de": "Furcht"},
+    "happy": {"tr": "Mutluluk", "de": "Glucklich"},
+    "sad": {"tr": "Uzuntu", "de": "Traurig"},
+    "surprise": {"tr": "Saskinlik", "de": "Uberraschung"},
+    "neutral": {"tr": "Notr", "de": "Neutral"},
+}

fer.py

@@ -0,0 +1,351 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# MIT License
+#
+# Copyright (c) 2018 Justin Shenk
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# IMPORTANT:
+#
+# This code is derived from Iván de Paz Centeno's implementation of MTCNN
+# (https://github.com/ipazc/mtcnn/) and Octavia Arriaga's facial expression recognition repo
+# (https://github.com/oarriaga/face_classification).
+#
+import logging
+import os
+import pkg_resources
+import requests
+import sys
+from typing import Sequence, Tuple, Union
+
+import cv2
+import numpy as np
+
+from tensorflow.keras.models import load_model
+
+
+from .utils import load_image
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger("fer")
+
+NumpyRects = Union[np.ndarray, Sequence[Tuple[int, int, int, int]]]
+
+__author__ = "Justin Shenk"
+
+PADDING = 40
+SERVER_URL = "http://localhost:8501/v1/models/emotion_model:predict"
+
+
+class FER(object):
+    """
+    Allows performing Facial Expression Recognition ->
+        a) Detection of faces
+        b) Detection of emotions
+    """
+
+    def __init__(
+        self,
+        cascade_file: str = None,
+        mtcnn=False,
+        tfserving: bool = False,
+        scale_factor: float = 1.1,
+        min_face_size: int = 50,
+        min_neighbors: int = 5,
+        offsets: tuple = (10, 10),
+    ):
+        """
+        Initializes the face detector and Keras model for facial expression recognition.
+        :param cascade_file: file URI with the Haar cascade for face classification
+        :param mtcnn: use MTCNN network for face detection (not yet implemented)
+        :param scale_factor: parameter specifying how much the image size is reduced at each image scale
+        :param min_face_size: minimum size of the face to detect
+        :param offsets: padding around face before classification
+        """
+        self.__scale_factor = scale_factor
+        self.__min_neighbors = min_neighbors
+        self.__min_face_size = min_face_size
+        self.__offsets = offsets
+        self.tfserving = tfserving
+
+        if cascade_file is None:
+            cascade_file = cv2.data.haarcascades + "haarcascade_frontalface_default.xml"
+
+        if mtcnn:
+            try:
+                from facenet_pytorch import MTCNN
+            except ImportError:
+                raise Exception(
+                    "MTCNN not installed, install it with pip install facenet-pytorch and from facenet_pytorch import MTCNN"
+                )
+            self.__face_detector = "mtcnn"
+            self._mtcnn = MTCNN(keep_all=True)
+        else:
+            self.__face_detector = cv2.CascadeClassifier(cascade_file)
+
+        self._initialize_model()
+
+    def _initialize_model(self):
+        if self.tfserving:
+            self.__emotion_target_size = (64, 64)  # hardcoded for now
+        else:
+            # Local Keras model
+            emotion_model = pkg_resources.resource_filename(
+                "fer", "data/emotion_model.hdf5"
+            )
+            log.debug("Emotion model: {}".format(emotion_model))
+            self.__emotion_classifier = load_model(emotion_model, compile=False)
+            self.__emotion_classifier.make_predict_function()
+            self.__emotion_target_size = self.__emotion_classifier.input_shape[1:3]
+        return
+
+    def _classify_emotions(self, gray_faces: np.ndarray) -> np.ndarray:  # b x w x h
+        """Run faces through online or offline classifier."""
+        if self.tfserving:
+            gray_faces = np.expand_dims(gray_faces, -1)  # to 4-dimensions
+            instances = gray_faces.tolist()
+            response = requests.post(SERVER_URL, json={"instances": instances})
+            response.raise_for_status()
+
+            emotion_predictions = response.json()["predictions"]
+            return emotion_predictions
+        else:
+            return self.__emotion_classifier(gray_faces)
+
+    @staticmethod
+    def pad(image):
+        """Pad image."""
+        row, col = image.shape[:2]
+        bottom = image[row - 2 : row, 0:col]
+        mean = cv2.mean(bottom)[0]
+
+        padded_image = cv2.copyMakeBorder(
+            image,
+            top=PADDING,
+            bottom=PADDING,
+            left=PADDING,
+            right=PADDING,
+            borderType=cv2.BORDER_CONSTANT,
+            value=[mean, mean, mean],
+        )
+        return padded_image
+
+    @staticmethod
+    def depad(image):
+        row, col = image.shape[:2]
+        return image[PADDING : row - PADDING, PADDING : col - PADDING]
+
+    @staticmethod
+    def tosquare(bbox):
+        """Convert bounding box to square by elongating shorter side."""
+        x, y, w, h = bbox
+        if h > w:
+            diff = h - w
+            x -= diff // 2
+            w += diff
+        elif w > h:
+            diff = w - h
+            y -= diff // 2
+            h += diff
+        if w != h:
+            log.debug(f"{w} is not {h}")
+
+        return (x, y, w, h)
+
+    def find_faces(self, img: np.ndarray, bgr=True) -> list:
+        """Image to list of faces bounding boxes(x,y,w,h)"""
+        if isinstance(self.__face_detector, cv2.CascadeClassifier):
+            if bgr:
+                gray_image_array = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+            else:  # assume gray
+                gray_image_array = img
+
+            faces = self.__face_detector.detectMultiScale(
+                gray_image_array,
+                scaleFactor=self.__scale_factor,
+                minNeighbors=self.__min_neighbors,
+                flags=cv2.CASCADE_SCALE_IMAGE,
+                minSize=(self.__min_face_size, self.__min_face_size),
+            )
+        elif self.__face_detector == "mtcnn":
+            boxes, probs = self._mtcnn.detect(img)
+            faces = []
+            if type(boxes) == np.ndarray:
+                for face in boxes:
+                    faces.append(
+                        [
+                            int(face[0]),
+                            int(face[1]),
+                            int(face[2]) - int(face[0]),
+                            int(face[3]) - int(face[1]),
+                        ]
+                    )
+
+        return faces
+
+    @staticmethod
+    def __preprocess_input(x, v2=False):
+        x = x.astype("float32")
+        x = x / 255.0
+        if v2:
+            x = x - 0.5
+            x = x * 2.0
+        return x
+
+    def __apply_offsets(self, face_coordinates):
+        """Offset face coordinates with padding before classification.
+        x1, x2, y1, y2 = 0, 100, 0, 100 becomes -10, 110, -10, 110
+        """
+        x, y, width, height = face_coordinates
+        x_off, y_off = self.__offsets
+        x1 = x - x_off
+        x2 = x + width + x_off
+        y1 = y - y_off
+        y2 = y + height + y_off
+        return x1, x2, y1, y2
+
+    @staticmethod
+    def _get_labels():
+        return {
+            0: "angry",
+            1: "disgust",
+            2: "fear",
+            3: "happy",
+            4: "sad",
+            5: "surprise",
+            6: "neutral",
+        }
+
+    def detect_emotions(
+        self, img: np.ndarray, face_rectangles: NumpyRects = None
+    ) -> list:
+        """
+        Detects bounding boxes from the specified image with ranking of emotions.
+        :param img: exact image path, numpy array (BGR or gray) or based64 encoded images
+        could be passed.
+        :return: list containing all the bounding boxes detected with their emotions.
+        """
+        img = load_image(img)
+
+        emotion_labels = self._get_labels()
+
+        if face_rectangles is None:
+            face_rectangles = self.find_faces(img, bgr=True)
+
+        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+        gray_img = self.pad(gray_img)
+
+        emotions = []
+        gray_faces = []
+        if face_rectangles is not None:
+            for face_coordinates in face_rectangles:
+                face_coordinates = self.tosquare(face_coordinates)
+
+                # offset to expand bounding box
+                # Note: x1 and y1 can be negative
+                x1, x2, y1, y2 = self.__apply_offsets(face_coordinates)
+
+                # account for padding in bounding box coordinates
+                x1 += PADDING
+                y1 += PADDING
+                x2 += PADDING
+                y2 += PADDING
+                x1 = np.clip(x1, a_min=0, a_max=None)
+                y1 = np.clip(y1, a_min=0, a_max=None)
+
+                gray_face = gray_img[max(0, y1) : y2, max(0, x1) : x2]
+
+                try:
+                    gray_face = cv2.resize(gray_face, self.__emotion_target_size)
+                except Exception as e:
+                    log.warn("{} resize failed: {}".format(gray_face.shape, e))
+                    continue
+
+                # Local Keras model
+                gray_face = self.__preprocess_input(gray_face, True)
+                gray_faces.append(gray_face)
+
+        # predict all faces
+        if not len(gray_faces):
+            return emotions  # no valid faces
+
+        # classify emotions
+        emotion_predictions = self._classify_emotions(np.array(gray_faces))
+
+        # label scores
+        for face_idx, face in enumerate(emotion_predictions):
+            labelled_emotions = {
+                emotion_labels[idx]: round(float(score), 2)
+                for idx, score in enumerate(face)
+            }
+
+            emotions.append(
+                dict(box=face_rectangles[face_idx], emotions=labelled_emotions)
+            )
+
+        self.emotions = emotions
+
+        return emotions
+
+    def top_emotion(
+        self, img: np.ndarray
+    ) -> Tuple[Union[str, None], Union[float, None]]:
+        """Convenience wrapper for `detect_emotions` returning only top emotion for first face in frame.
+        :param img: image to process
+        :return: top emotion and score (for first face in frame) or (None, None)
+
+        """
+        emotions = self.detect_emotions(img=img)
+        top_emotions = [
+            max(e["emotions"], key=lambda key: e["emotions"][key]) for e in emotions
+        ]
+
+        # Take first face
+        if len(top_emotions):
+            top_emotion = top_emotions[0]
+        else:
+            return (None, None)
+        score = emotions[0]["emotions"][top_emotion]
+
+        return top_emotion, score
+
+
+def parse_arguments(args):
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--image", type=str, help="Image filepath")
+    return parser.parse_args()
+
+
+def top_emotion():
+    args = parse_arguments(sys.argv)
+    fer = FER()
+    top_emotion, score = fer.top_emotion(args.image)
+    print(top_emotion, score)
+
+
+def main():
+    top_emotion()
+
+
+if __name__ == "__main__":
+    main()

utils.py

@@ -0,0 +1,116 @@
+import base64
+import os
+import requests
+
+import cv2
+import numpy as np
+from PIL import Image
+
+from .exceptions import InvalidImage
+from .emotionsmultilanguage import emotions_dict
+
+
+def draw_annotations(
+    frame: np.ndarray,
+    faces: list,
+    boxes=True,
+    scores=True,
+    color: tuple = (0, 155, 255),
+    lang: str = "en",
+    size_multiplier: int = 1,
+) -> np.ndarray:
+    """Draws boxes around detected faces. Faces is a list of dicts with `box` and `emotions`."""
+    if not len(faces):
+        return frame
+
+    for face in faces:
+        x, y, w, h = face["box"]
+        emotions = face["emotions"]
+
+        if boxes:
+            cv2.rectangle(
+                frame,
+                (x, y, w, h),
+                color,
+                2,
+            )
+
+        if scores:
+            frame = draw_scores(frame, emotions, (x, y, w, h), lang, size_multiplier)
+    return frame
+
+
+def loadBase64Img(uri):
+    encoded_data = uri.split(",")[1]
+    nparr = np.fromstring(base64.b64decode(encoded_data), np.uint8)
+    img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+    return img
+
+
+def pil_to_bgr(pil_image):
+    return cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR)
+
+
+def load_image(img):
+    """Modified from github.com/serengil/deepface. Returns bgr (opencv-style) numpy array."""
+    is_exact_image = is_base64_img = is_url_img = False
+
+    if type(img).__module__ == np.__name__:
+        is_exact_image = True
+    elif img is None:
+        raise InvalidImage("Image not valid.")
+    elif len(img) > 11 and img[0:11] == "data:image/":
+        is_base64_img = True
+    elif len(img) > 11 and img.startswith("http"):
+        is_url_img = True
+
+    if is_base64_img:
+        img = loadBase64Img(img)
+    elif is_url_img:
+        img = pil_to_bgr(Image.open(requests.get(img, stream=True).raw))
+    elif not is_exact_image:  # image path passed as input
+        if not os.path.isfile(img):
+            raise ValueError(f"Confirm that {img} exists")
+        img = cv2.imread(img)
+
+    if img is None or not hasattr(img, "shape"):
+        raise InvalidImage("Image not valid.")
+
+    return img
+
+
+def draw_scores(
+    frame: np.ndarray,
+    emotions: dict,
+    bounding_box: dict,
+    lang: str = "en",
+    size_multiplier: int = 1,
+) -> np.ndarray:
+    """Draw scores for each emotion under faces."""
+    GRAY = (211, 211, 211)
+    GREEN = (0, 255, 0)
+    x, y, w, h = bounding_box
+
+    for idx, (emotion, score) in enumerate(emotions.items()):
+        color = GRAY if score < 0.01 else GREEN
+
+        if lang != "en":
+            emotion = emotions_dict[emotion][lang]
+
+        emotion_score = "{}: {}".format(
+            emotion, "{:.2f}".format(score) if score >= 0.01 else ""
+        )
+        cv2.putText(
+            frame,
+            emotion_score,
+            (
+                x,
+                y + h + (15 * size_multiplier) + idx * (15 * size_multiplier),
+            ),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.5 * size_multiplier,
+            color,
+            1 * size_multiplier,
+            cv2.LINE_AA,
+        )
+    return frame