Upload 13 files

Exercise_Recognition_AI-main/.dockerignore

@@ -0,0 +1,14 @@
+.git
+.ipynb_checkpoints
+data
+logs
+old
+research
+tests
+LICENSE
+README.md
+*.mp4
+*.png
+*.h5
+!models/*.h5
+.gitignore

Exercise_Recognition_AI-main/.github/workflows/CI.yml

@@ -0,0 +1,24 @@
+name: Build Docker image and deploy to Heroku
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Login to Heroku Container registry
+        env:
+          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+        run: heroku container:login
+      - name: Build and push
+        env:
+          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+        run: heroku container:push -a ai-personal-fitness-trainer web
+      - name: Release
+        env:
+          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+        run: heroku container:release -a ai-personal-fitness-trainer web

Exercise_Recognition_AI-main/.gitignore

@@ -0,0 +1,11 @@
+*.egg-info
+*.pyc
+data
+old
+logs
+.ipynb_checkpoints
+*.h5
+!models/*.h5
+env
+*.avi
+*.mp4

Exercise_Recognition_AI-main/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.8
+EXPOSE 8501
+WORKDIR /app
+COPY requirements.txt ./requirements.txt
+RUN apt-get update
+RUN apt-get install ffmpeg libsm6 libxext6  -y
+RUN pip3 install -r requirements.txt
+COPY . .
+CMD streamlit run --server.port $PORT app.py

Exercise_Recognition_AI-main/app.py

@@ -0,0 +1,378 @@
+import streamlit as st
+import cv2
+
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import (LSTM, Dense, Dropout, Input, Flatten, 
+                                     Bidirectional, Permute, multiply)
+
+import numpy as np
+import mediapipe as mp
+import math
+
+from streamlit_webrtc import webrtc_streamer, WebRtcMode, RTCConfiguration
+import av
+
+## Build and Load Model
+def attention_block(inputs, time_steps):
+    """
+    Attention layer for deep neural network
+    
+    """
+    # Attention weights
+    a = Permute((2, 1))(inputs)
+    a = Dense(time_steps, activation='softmax')(a)
+    
+    # Attention vector
+    a_probs = Permute((2, 1), name='attention_vec')(a)
+    
+    # Luong's multiplicative score
+    output_attention_mul = multiply([inputs, a_probs], name='attention_mul') 
+    
+    return output_attention_mul
+
+@st.cache(allow_output_mutation=True)
+def build_model(HIDDEN_UNITS=256, sequence_length=30, num_input_values=33*4, num_classes=3):
+    """
+    Function used to build the deep neural network model on startup
+
+    Args:
+        HIDDEN_UNITS (int, optional): Number of hidden units for each neural network hidden layer. Defaults to 256.
+        sequence_length (int, optional): Input sequence length (i.e., number of frames). Defaults to 30.
+        num_input_values (_type_, optional): Input size of the neural network model. Defaults to 33*4 (i.e., number of keypoints x number of metrics).
+        num_classes (int, optional): Number of classification categories (i.e., model output size). Defaults to 3.
+
+    Returns:
+        keras model: neural network with pre-trained weights
+    """
+    # Input
+    inputs = Input(shape=(sequence_length, num_input_values))
+    # Bi-LSTM
+    lstm_out = Bidirectional(LSTM(HIDDEN_UNITS, return_sequences=True))(inputs)
+    # Attention
+    attention_mul = attention_block(lstm_out, sequence_length)
+    attention_mul = Flatten()(attention_mul)
+    # Fully Connected Layer
+    x = Dense(2*HIDDEN_UNITS, activation='relu')(attention_mul)
+    x = Dropout(0.5)(x)
+    # Output
+    x = Dense(num_classes, activation='softmax')(x)
+    # Bring it all together
+    model = Model(inputs=[inputs], outputs=x)
+
+    ## Load Model Weights
+    load_dir = "./models/LSTM_Attention.h5"  
+    model.load_weights(load_dir)
+    
+    return model
+
+HIDDEN_UNITS = 256
+model = build_model(HIDDEN_UNITS)
+
+## App
+st.write("# AI Personal Fitness Trainer Web App")
+
+st.markdown("❗❗ **Development Note** ❗❗")
+st.markdown("Currently, the exercise recognition model uses the the x, y, and z coordinates of each anatomical landmark from the MediaPipe Pose model. These coordinates are normalized with respect to the image frame (e.g., the top left corner represents (x=0,y=0) and the bottom right corner represents(x=1,y=1)).")
+st.markdown("I'm currently developing and testing two new feature engineering strategies:")
+st.markdown("- Normalizing coordinates by the detected bounding box of the user")
+st.markdown("- Using joint angles rather than keypoint coordaintes as features")            
+st.write("Stay Tuned!")
+
+st.write("## Settings")
+threshold1 = st.slider("Minimum Keypoint Detection Confidence", 0.00, 1.00, 0.50)
+threshold2 = st.slider("Minimum Tracking Confidence", 0.00, 1.00, 0.50)
+threshold3 = st.slider("Minimum Activity Classification Confidence", 0.00, 1.00, 0.50)
+
+st.write("## Activate the AI 🤖🏋️‍♂️")
+
+## Mediapipe
+mp_pose = mp.solutions.pose # Pre-trained pose estimation model from Google Mediapipe
+mp_drawing = mp.solutions.drawing_utils # Supported Mediapipe visualization tools
+pose = mp_pose.Pose(min_detection_confidence=threshold1, min_tracking_confidence=threshold2) # mediapipe pose model
+
+## Real Time Machine Learning and Computer Vision Processes
+class VideoProcessor:
+    def __init__(self):
+        # Parameters
+        self.actions = np.array(['curl', 'press', 'squat'])
+        self.sequence_length = 30
+        self.colors = [(245,117,16), (117,245,16), (16,117,245)]
+        self.threshold = threshold3
+        
+        # Detection variables
+        self.sequence = []
+        self.current_action = ''
+
+        # Rep counter logic variables
+        self.curl_counter = 0
+        self.press_counter = 0
+        self.squat_counter = 0
+        self.curl_stage = None
+        self.press_stage = None
+        self.squat_stage = None
+    
+    @st.cache()    
+    def draw_landmarks(self, image, results):
+        """
+        This function draws keypoints and landmarks detected by the human pose estimation model
+        
+        """
+        mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,
+                                    mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2), 
+                                    mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2) 
+                                    )
+        return
+    
+    @st.cache()
+    def extract_keypoints(self, results):
+        """
+        Processes and organizes the keypoints detected from the pose estimation model 
+        to be used as inputs for the exercise decoder models
+        
+        """
+        pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)
+        return pose
+    
+    @st.cache()
+    def calculate_angle(self, a,b,c):
+        """
+        Computes 3D joint angle inferred by 3 keypoints and their relative positions to one another
+        
+        """
+        a = np.array(a) # First
+        b = np.array(b) # Mid
+        c = np.array(c) # End
+        
+        radians = np.arctan2(c[1]-b[1], c[0]-b[0]) - np.arctan2(a[1]-b[1], a[0]-b[0])
+        angle = np.abs(radians*180.0/np.pi)
+        
+        if angle > 180.0:
+            angle = 360-angle
+            
+        return angle 
+    
+    @st.cache()
+    def get_coordinates(self, landmarks, mp_pose, side, joint):
+        """
+        Retrieves x and y coordinates of a particular keypoint from the pose estimation model
+            
+        Args:
+            landmarks: processed keypoints from the pose estimation model
+            mp_pose: Mediapipe pose estimation model
+            side: 'left' or 'right'. Denotes the side of the body of the landmark of interest.
+            joint: 'shoulder', 'elbow', 'wrist', 'hip', 'knee', or 'ankle'. Denotes which body joint is associated with the landmark of interest.
+        
+        """
+        coord = getattr(mp_pose.PoseLandmark,side.upper()+"_"+joint.upper())
+        x_coord_val = landmarks[coord.value].x
+        y_coord_val = landmarks[coord.value].y
+        return [x_coord_val, y_coord_val] 
+    
+    @st.cache()
+    def viz_joint_angle(self, image, angle, joint):
+        """
+        Displays the joint angle value near the joint within the image frame
+        
+        """
+        cv2.putText(image, str(int(angle)), 
+                    tuple(np.multiply(joint, [640, 480]).astype(int)), 
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA
+                            )
+        return
+    
+    @st.cache()
+    def count_reps(self, image, landmarks, mp_pose):
+        """
+        Counts repetitions of each exercise. Global count and stage (i.e., state) variables are updated within this function.
+        
+        """
+        
+        if self.current_action == 'curl':
+            # Get coords
+            shoulder = self.get_coordinates(landmarks, mp_pose, 'left', 'shoulder')
+            elbow = self.get_coordinates(landmarks, mp_pose, 'left', 'elbow')
+            wrist = self.get_coordinates(landmarks, mp_pose, 'left', 'wrist')
+            
+            # calculate elbow angle
+            angle = self.calculate_angle(shoulder, elbow, wrist)
+            
+            # curl counter logic
+            if angle < 30:
+                self.curl_stage = "up" 
+            if angle > 140 and self.curl_stage =='up':
+                self.curl_stage="down"  
+                self.curl_counter +=1
+            self.press_stage = None
+            self.squat_stage = None
+                
+            # Viz joint angle
+            self.viz_joint_angle(image, angle, elbow)
+            
+        elif self.current_action == 'press':           
+            # Get coords
+            shoulder = self.get_coordinates(landmarks, mp_pose, 'left', 'shoulder')
+            elbow = self.get_coordinates(landmarks, mp_pose, 'left', 'elbow')
+            wrist = self.get_coordinates(landmarks, mp_pose, 'left', 'wrist')
+
+            # Calculate elbow angle
+            elbow_angle = self.calculate_angle(shoulder, elbow, wrist)
+            
+            # Compute distances between joints
+            shoulder2elbow_dist = abs(math.dist(shoulder,elbow))
+            shoulder2wrist_dist = abs(math.dist(shoulder,wrist))
+            
+            # Press counter logic
+            if (elbow_angle > 130) and (shoulder2elbow_dist < shoulder2wrist_dist):
+                self.press_stage = "up"
+            if (elbow_angle < 50) and (shoulder2elbow_dist > shoulder2wrist_dist) and (self.press_stage =='up'):
+                self.press_stage='down'
+                self.press_counter += 1
+            self.curl_stage = None
+            self.squat_stage = None
+                
+            # Viz joint angle
+            self.viz_joint_angle(image, elbow_angle, elbow)
+            
+        elif self.current_action == 'squat':
+            # Get coords
+            # left side
+            left_shoulder = self.get_coordinates(landmarks, mp_pose, 'left', 'shoulder')
+            left_hip = self.get_coordinates(landmarks, mp_pose, 'left', 'hip')
+            left_knee = self.get_coordinates(landmarks, mp_pose, 'left', 'knee')
+            left_ankle = self.get_coordinates(landmarks, mp_pose, 'left', 'ankle')
+            # right side
+            right_shoulder = self.get_coordinates(landmarks, mp_pose, 'right', 'shoulder')
+            right_hip = self.get_coordinates(landmarks, mp_pose, 'right', 'hip')
+            right_knee = self.get_coordinates(landmarks, mp_pose, 'right', 'knee')
+            right_ankle = self.get_coordinates(landmarks, mp_pose, 'right', 'ankle')
+            
+            # Calculate knee angles
+            left_knee_angle = self.calculate_angle(left_hip, left_knee, left_ankle)
+            right_knee_angle = self.calculate_angle(right_hip, right_knee, right_ankle)
+            
+            # Calculate hip angles
+            left_hip_angle = self.calculate_angle(left_shoulder, left_hip, left_knee)
+            right_hip_angle = self.calculate_angle(right_shoulder, right_hip, right_knee)
+            
+            # Squat counter logic
+            thr = 165
+            if (left_knee_angle < thr) and (right_knee_angle < thr) and (left_hip_angle < thr) and (right_hip_angle < thr):
+                self.squat_stage = "down"
+            if (left_knee_angle > thr) and (right_knee_angle > thr) and (left_hip_angle > thr) and (right_hip_angle > thr) and (self.squat_stage =='down'):
+                self.squat_stage='up'
+                self.squat_counter += 1
+            self.curl_stage = None
+            self.press_stage = None
+                
+            # Viz joint angles
+            self.viz_joint_angle(image, left_knee_angle, left_knee)
+            self.viz_joint_angle(image, left_hip_angle, left_hip)
+            
+        else:
+            pass
+        return
+    
+    @st.cache()
+    def prob_viz(self, res, input_frame):
+        """
+        This function displays the model prediction probability distribution over the set of exercise classes
+        as a horizontal bar graph
+        
+        """
+        output_frame = input_frame.copy()
+        for num, prob in enumerate(res):        
+            cv2.rectangle(output_frame, (0,60+num*40), (int(prob*100), 90+num*40), self.colors[num], -1)
+            cv2.putText(output_frame, self.actions[num], (0, 85+num*40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.LINE_AA)
+            
+        return output_frame
+    
+    @st.cache()
+    def process(self, image):
+        """
+        Function to process the video frame from the user's webcam and run the fitness trainer AI
+
+        Args:
+            image (numpy array): input image from the webcam
+
+        Returns:
+            numpy array: processed image with keypoint detection and fitness activity classification visualized
+        """
+        # Pose detection model
+        image.flags.writeable = False
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        results = pose.process(image)
+
+        # Draw the hand annotations on the image.
+        image.flags.writeable = True
+        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+        self.draw_landmarks(image, results) 
+        
+        # Prediction logic
+        keypoints = self.extract_keypoints(results)        
+        self.sequence.append(keypoints.astype('float32',casting='same_kind'))      
+        self.sequence = self.sequence[-self.sequence_length:]
+        
+        if len(self.sequence) == self.sequence_length:
+            res = model.predict(np.expand_dims(self.sequence, axis=0), verbose=0)[0]
+            # interpreter.set_tensor(self.input_details[0]['index'], np.expand_dims(self.sequence, axis=0))
+            # interpreter.invoke()
+            # res = interpreter.get_tensor(self.output_details[0]['index'])
+            
+            self.current_action = self.actions[np.argmax(res)]
+            confidence = np.max(res)
+            
+            # Erase current action variable if no probability is above threshold
+            if confidence < self.threshold:
+                self.current_action = ''
+
+            # Viz probabilities
+            image = self.prob_viz(res, image)
+            
+            # Count reps
+            try:
+                landmarks = results.pose_landmarks.landmark
+                self.count_reps(
+                    image, landmarks, mp_pose)
+            except:
+                pass
+
+            # Display graphical information
+            cv2.rectangle(image, (0,0), (640, 40), self.colors[np.argmax(res)], -1)
+            cv2.putText(image, 'curl ' + str(self.curl_counter), (3,30), 
+                           cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
+            cv2.putText(image, 'press ' + str(self.press_counter), (240,30), 
+                           cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
+            cv2.putText(image, 'squat ' + str(self.squat_counter), (490,30), 
+                           cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
+          
+        # return cv2.flip(image, 1)
+        return image
+    
+    def recv(self, frame):
+        """
+        Receive and process video stream from webcam
+
+        Args:
+            frame: current video frame
+
+        Returns:
+            av.VideoFrame: processed video frame
+        """
+        img = frame.to_ndarray(format="bgr24")
+        img = self.process(img)
+        return av.VideoFrame.from_ndarray(img, format="bgr24")
+        
+## Stream Webcam Video and Run Model
+# Options
+RTC_CONFIGURATION = RTCConfiguration(
+    {"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]}
+)
+# Streamer
+webrtc_ctx = webrtc_streamer(
+    key="AI trainer",
+    mode=WebRtcMode.SENDRECV,
+    rtc_configuration=RTC_CONFIGURATION,
+    media_stream_constraints={"video": True, "audio": False},
+    video_processor_factory=VideoProcessor,
+    async_processing=True,
+)

Exercise_Recognition_AI-main/environment.yml

@@ -0,0 +1,266 @@
+name: AItrainer
+channels:
+  - conda-forge
+  - anaconda
+  - soumith
+  - defaults
+dependencies:
+  - _tflow_select=2.3.0=gpu
+  - aiohttp=3.8.1=py38h294d835_1
+  - aiosignal=1.2.0=pyhd8ed1ab_0
+  - alabaster=0.7.12=pyhd3eb1b0_0
+  - appdirs=1.4.4=pyhd3eb1b0_0
+  - argon2-cffi=21.3.0=pyhd3eb1b0_0
+  - argon2-cffi-bindings=21.2.0=py38h2bbff1b_0
+  - arrow=1.2.2=pyhd3eb1b0_0
+  - astor=0.8.1=pyh9f0ad1d_0
+  - astroid=2.9.0=py38haa95532_0
+  - asttokens=2.0.5=pyhd3eb1b0_0
+  - astunparse=1.6.3=pyhd8ed1ab_0
+  - async-timeout=4.0.2=pyhd8ed1ab_0
+  - atomicwrites=1.4.0=py_0
+  - attrs=21.4.0=pyhd3eb1b0_0
+  - autopep8=1.5.6=pyhd3eb1b0_0
+  - babel=2.9.1=pyhd3eb1b0_0
+  - backcall=0.2.0=pyhd3eb1b0_0
+  - bcrypt=3.2.0=py38he774522_0
+  - beautifulsoup4=4.11.1=py38haa95532_0
+  - binaryornot=0.4.4=pyhd3eb1b0_1
+  - black=19.10b0=py_0
+  - blas=1.0=mkl
+  - bleach=4.1.0=pyhd3eb1b0_0
+  - blinker=1.4=py_1
+  - brotlipy=0.7.0=py38h2bbff1b_1003
+  - ca-certificates=2022.6.15=h5b45459_0
+  - cachetools=5.0.0=pyhd8ed1ab_0
+  - certifi=2022.6.15=py38haa244fe_0
+  - cffi=1.15.0=py38h2bbff1b_1
+  - chardet=4.0.0=py38haa95532_1003
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - click=8.0.4=py38haa95532_0
+  - cloudpickle=2.0.0=pyhd3eb1b0_0
+  - colorama=0.4.4=pyhd3eb1b0_0
+  - cookiecutter=1.7.3=pyhd3eb1b0_0
+  - cryptography=37.0.1=py38h21b164f_0
+  - debugpy=1.5.1=py38hd77b12b_0
+  - decorator=5.1.1=pyhd3eb1b0_0
+  - defusedxml=0.7.1=pyhd3eb1b0_0
+  - diff-match-patch=20200713=pyhd3eb1b0_0
+  - docutils=0.17.1=py38haa95532_1
+  - eigen=3.3.7=h59b6b97_1
+  - entrypoints=0.4=py38haa95532_0
+  - executing=0.8.3=pyhd3eb1b0_0
+  - flake8=3.9.0=pyhd3eb1b0_0
+  - frozenlist=1.3.0=py38h294d835_1
+  - future=0.18.2=py38_1
+  - gast=0.4.0=pyh9f0ad1d_0
+  - glib=2.69.1=h5dc1a3c_1
+  - google-auth=2.8.0=pyh6c4a22f_0
+  - google-auth-oauthlib=0.4.6=pyhd8ed1ab_0
+  - google-pasta=0.2.0=pyh8c360ce_0
+  - gst-plugins-base=1.18.5=h9e645db_0
+  - gstreamer=1.18.5=hd78058f_0
+  - h5py=2.10.0=py38h5e291fa_0
+  - hdf5=1.10.4=h7ebc959_0
+  - icc_rt=2019.0.0=h0cc432a_1
+  - icu=58.2=ha925a31_3
+  - idna=3.3=pyhd3eb1b0_0
+  - imagesize=1.3.0=pyhd3eb1b0_0
+  - importlib-metadata=4.11.3=py38haa95532_0
+  - importlib_metadata=4.11.3=hd3eb1b0_0
+  - importlib_resources=5.2.0=pyhd3eb1b0_1
+  - inflection=0.5.1=py38haa95532_0
+  - intel-openmp=2021.4.0=haa95532_3556
+  - intervaltree=3.1.0=pyhd3eb1b0_0
+  - ipykernel=6.9.1=py38haa95532_0
+  - ipython=8.3.0=py38haa95532_0
+  - ipython_genutils=0.2.0=pyhd3eb1b0_1
+  - isort=5.9.3=pyhd3eb1b0_0
+  - jedi=0.17.2=py38haa95532_1
+  - jinja2=3.0.3=pyhd3eb1b0_0
+  - jinja2-time=0.2.0=pyhd3eb1b0_3
+  - joblib=1.1.0=pyhd3eb1b0_0
+  - jpeg=9e=h2bbff1b_0
+  - jsonschema=4.4.0=py38haa95532_0
+  - jupyter_client=7.2.2=py38haa95532_0
+  - jupyter_core=4.10.0=py38haa95532_0
+  - jupyterlab_pygments=0.1.2=py_0
+  - keras-applications=1.0.8=py_1
+  - keras-preprocessing=1.1.2=pyhd8ed1ab_0
+  - keyring=23.4.0=py38haa95532_0
+  - lazy-object-proxy=1.6.0=py38h2bbff1b_0
+  - libblas=3.9.0=1_h8933c1f_netlib
+  - libcblas=3.9.0=5_hd5c7e75_netlib
+  - libffi=3.4.2=hd77b12b_4
+  - libiconv=1.16=h2bbff1b_2
+  - liblapack=3.9.0=5_hd5c7e75_netlib
+  - libogg=1.3.5=h2bbff1b_1
+  - libopencv=4.0.1=hbb9e17c_0
+  - libpng=1.6.37=h2a8f88b_0
+  - libprotobuf=3.20.1=h23ce68f_0
+  - libspatialindex=1.9.3=h6c2663c_0
+  - libtiff=4.2.0=he0120a3_1
+  - libvorbis=1.3.7=he774522_0
+  - libwebp-base=1.2.2=h2bbff1b_0
+  - lz4-c=1.9.3=h2bbff1b_1
+  - m2w64-gcc-libgfortran=5.3.0=6
+  - m2w64-gcc-libs=5.3.0=7
+  - m2w64-gcc-libs-core=5.3.0=7
+  - m2w64-gmp=6.1.0=2
+  - m2w64-libwinpthread-git=5.0.0.4634.697f757=2
+  - markdown=3.3.7=pyhd8ed1ab_0
+  - markupsafe=2.1.1=py38h2bbff1b_0
+  - matplotlib-inline=0.1.2=pyhd3eb1b0_2
+  - mccabe=0.6.1=py38_1
+  - mistune=0.8.4=py38he774522_1000
+  - msys2-conda-epoch=20160418=1
+  - multidict=6.0.2=py38h294d835_1
+  - mypy_extensions=0.4.3=py38haa95532_1
+  - nbclient=0.5.13=py38haa95532_0
+  - nbconvert=6.4.4=py38haa95532_0
+  - nbformat=5.3.0=py38haa95532_0
+  - nest-asyncio=1.5.5=py38haa95532_0
+  - nomkl=1.0=h5ca1d4c_0
+  - notebook=6.4.11=py38haa95532_0
+  - numpydoc=1.2=pyhd3eb1b0_0
+  - oauthlib=3.2.0=pyhd8ed1ab_0
+  - opencv=4.0.1=py38h2a7c758_0
+  - openssl=1.1.1p=h8ffe710_0
+  - opt_einsum=3.3.0=pyhd8ed1ab_1
+  - packaging=21.3=pyhd3eb1b0_0
+  - pandas=1.2.4=py38hf11a4ad_0
+  - pandocfilters=1.5.0=pyhd3eb1b0_0
+  - paramiko=2.8.1=pyhd3eb1b0_0
+  - parso=0.7.0=py_0
+  - pathspec=0.7.0=py_0
+  - pcre=8.45=hd77b12b_0
+  - pexpect=4.8.0=pyhd3eb1b0_3
+  - pickleshare=0.7.5=pyhd3eb1b0_1003
+  - pip=21.2.2=py38haa95532_0
+  - platformdirs=2.4.0=pyhd3eb1b0_0
+  - pluggy=1.0.0=py38haa95532_1
+  - poyo=0.5.0=pyhd3eb1b0_0
+  - prometheus_client=0.13.1=pyhd3eb1b0_0
+  - prompt-toolkit=3.0.20=pyhd3eb1b0_0
+  - psutil=5.8.0=py38h2bbff1b_1
+  - ptyprocess=0.7.0=pyhd3eb1b0_2
+  - pure_eval=0.2.2=pyhd3eb1b0_0
+  - py-opencv=4.0.1=py38he44ac1e_0
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycodestyle=2.6.0=pyhd3eb1b0_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pydocstyle=6.1.1=pyhd3eb1b0_0
+  - pyflakes=2.2.0=pyhd3eb1b0_0
+  - pygments=2.11.2=pyhd3eb1b0_0
+  - pyjwt=2.4.0=pyhd8ed1ab_0
+  - pylint=2.12.2=py38haa95532_1
+  - pyls-black=0.4.6=hd3eb1b0_0
+  - pyls-spyder=0.3.2=pyhd3eb1b0_0
+  - pynacl=1.4.0=py38h62dcd97_1
+  - pyopenssl=22.0.0=pyhd3eb1b0_0
+  - pyqt=5.9.2=py38hd77b12b_6
+  - pyreadline=2.1=py38haa244fe_1005
+  - pyrsistent=0.18.0=py38h196d8e1_0
+  - pysocks=1.7.1=py38haa95532_0
+  - python=3.8.13=h6244533_0
+  - python-dateutil=2.8.2=pyhd3eb1b0_0
+  - python-fastjsonschema=2.15.1=pyhd3eb1b0_0
+  - python-jsonrpc-server=0.4.0=py_0
+  - python-language-server=0.36.2=pyhd3eb1b0_0
+  - python-slugify=5.0.2=pyhd3eb1b0_0
+  - python_abi=3.8=2_cp38
+  - pytz=2022.1=py38haa95532_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pywin32=302=py38h2bbff1b_2
+  - pywin32-ctypes=0.2.0=py38_1000
+  - pywinpty=2.0.2=py38h5da7b33_0
+  - pyyaml=6.0=py38h2bbff1b_1
+  - pyzmq=22.3.0=py38hd77b12b_2
+  - qdarkstyle=3.0.2=pyhd3eb1b0_0
+  - qstylizer=0.1.10=pyhd3eb1b0_0
+  - qt=5.9.7=vc14h73c81de_0
+  - qtawesome=1.0.3=pyhd3eb1b0_0
+  - qtconsole=5.3.0=pyhd3eb1b0_0
+  - qtpy=2.0.1=pyhd3eb1b0_0
+  - regex=2022.3.15=py38h2bbff1b_0
+  - requests=2.27.1=pyhd3eb1b0_0
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - rope=0.22.0=pyhd3eb1b0_0
+  - rsa=4.8=pyhd8ed1ab_0
+  - rtree=0.9.7=py38h2eaa2aa_1
+  - scikit-learn=1.0.2=py38hf11a4ad_1
+  - scipy=1.5.3=py38h5f893b4_0
+  - send2trash=1.8.0=pyhd3eb1b0_1
+  - setuptools=61.2.0=py38haa95532_0
+  - sip=4.19.13=py38hd77b12b_0
+  - snowballstemmer=2.2.0=pyhd3eb1b0_0
+  - sortedcontainers=2.4.0=pyhd3eb1b0_0
+  - soupsieve=2.3.1=pyhd3eb1b0_0
+  - sphinx=4.4.0=pyhd3eb1b0_0
+  - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0
+  - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0
+  - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0
+  - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0
+  - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0
+  - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0
+  - spyder=5.0.5=py38haa95532_2
+  - spyder-kernels=2.0.5=py38haa95532_0
+  - sqlite=3.38.3=h2bbff1b_0
+  - stack_data=0.2.0=pyhd3eb1b0_0
+  - tensorboard=2.9.0=pyhd8ed1ab_0
+  - tensorboard-data-server=0.6.0=py38haa244fe_2
+  - tensorboard-plugin-wit=1.8.1=pyhd8ed1ab_0
+  - tensorflow=2.3.0=mkl_py38h8557ec7_0
+  - tensorflow-base=2.3.0=eigen_py38h75a453f_0
+  - tensorflow-estimator=2.5.0=pyh8a188c0_0
+  - tensorflow-gpu=2.3.0=he13fc11_0
+  - termcolor=1.1.0=py_2
+  - terminado=0.13.1=py38haa95532_0
+  - testpath=0.5.0=pyhd3eb1b0_0
+  - text-unidecode=1.3=pyhd3eb1b0_0
+  - textdistance=4.2.1=pyhd3eb1b0_0
+  - threadpoolctl=2.2.0=pyh0d69192_0
+  - three-merge=0.1.1=pyhd3eb1b0_0
+  - tinycss=0.4=pyhd3eb1b0_1002
+  - toml=0.10.2=pyhd3eb1b0_0
+  - tornado=6.1=py38h2bbff1b_0
+  - traitlets=5.1.1=pyhd3eb1b0_0
+  - typed-ast=1.4.3=py38h2bbff1b_1
+  - ujson=5.1.0=py38hd77b12b_0
+  - unidecode=1.2.0=pyhd3eb1b0_0
+  - urllib3=1.26.9=py38haa95532_0
+  - vc=14.2=h21ff451_1
+  - vs2015_runtime=14.27.29016=h5e58377_2
+  - watchdog=2.1.6=py38haa95532_0
+  - wcwidth=0.2.5=pyhd3eb1b0_0
+  - webencodings=0.5.1=py38_1
+  - werkzeug=2.1.2=pyhd8ed1ab_1
+  - wheel=0.37.1=pyhd3eb1b0_0
+  - win_inet_pton=1.1.0=py38haa95532_0
+  - wincertstore=0.2=py38haa95532_2
+  - winpty=0.4.3=4
+  - xz=5.2.5=h8cc25b3_1
+  - yaml=0.2.5=he774522_0
+  - yapf=0.31.0=pyhd3eb1b0_0
+  - yarl=1.7.2=py38h294d835_2
+  - zipp=3.8.0=py38haa95532_0
+  - zlib=1.2.12=h8cc25b3_2
+  - zstd=1.5.2=h19a0ad4_0
+  - pip:
+    - absl-py==0.15.0
+    - cycler==0.11.0
+    - fonttools==4.33.3
+    - grpcio==1.32.0
+    - kiwisolver==1.4.3
+    - matplotlib==3.5.2
+    - mediapipe==0.8.10
+    - numpy==1.23.0
+    - opencv-contrib-python==4.6.0.66
+    - pillow==9.1.1
+    - protobuf==4.21.1
+    - pyparsing==3.0.9
+    - six==1.15.0
+    - typing-extensions==3.7.4.3
+    - wrapt==1.12.1
+prefix: C:\Users\cpras\anaconda3\envs\AItrainer

Exercise_Recognition_AI-main/main.py

@@ -0,0 +1,324 @@
+import os
+import streamlit as st
+import cv2
+import mediapipe as mp
+import numpy as np
+import math
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import (LSTM, Dense, Dropout, Input, Flatten, 
+                                     Bidirectional, Permute, multiply)
+
+# Load the pose estimation model from Mediapipe
+mp_pose = mp.solutions.pose 
+mp_drawing = mp.solutions.drawing_utils 
+pose = mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) 
+
+# Define the attention block for the LSTM model
+def attention_block(inputs, time_steps):
+    a = Permute((2, 1))(inputs)
+    a = Dense(time_steps, activation='softmax')(a)
+    a_probs = Permute((2, 1), name='attention_vec')(a)
+    output_attention_mul = multiply([inputs, a_probs], name='attention_mul') 
+    return output_attention_mul
+
+# Build and load the LSTM model
+@st.cache(allow_output_mutation=True)
+def build_model(HIDDEN_UNITS=256, sequence_length=30, num_input_values=33*4, num_classes=3):
+    inputs = Input(shape=(sequence_length, num_input_values))
+    lstm_out = Bidirectional(LSTM(HIDDEN_UNITS, return_sequences=True))(inputs)
+    attention_mul = attention_block(lstm_out, sequence_length)
+    attention_mul = Flatten()(attention_mul)
+    x = Dense(2*HIDDEN_UNITS, activation='relu')(attention_mul)
+    x = Dropout(0.5)(x)
+    x = Dense(num_classes, activation='softmax')(x)
+    model = Model(inputs=[inputs], outputs=x)
+    load_dir = "./models/LSTM_Attention.h5"  
+    model.load_weights(load_dir)
+    return model
+
+# Define the VideoProcessor class for real-time video processing
+class VideoProcessor:
+    def __init__(self):
+        # Parameters
+        self.actions = np.array(['curl', 'press', 'squat'])
+        self.sequence_length = 30
+        self.colors = [(245,117,16), (117,245,16), (16,117,245)]
+        self.threshold = 0.5
+
+        self.model = build_model(256)
+        
+        # Detection variables
+        self.sequence = []
+        self.current_action = ''
+
+        # Rep counter logic variables
+        self.curl_counter = 0
+        self.press_counter = 0
+        self.squat_counter = 0
+        self.curl_stage = None
+        self.press_stage = None
+        self.squat_stage = None
+        self.pose = mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5)
+
+    def process_video(self, video_file):
+        # Get the filename from the file object
+        filename = "temp_video.mp4"
+        # Create a temporary file to write the contents of the uploaded video file
+        with open(filename, 'wb') as temp_file:
+            temp_file.write(video_file.read())
+
+        # Process the video and save the processed video to a new file
+        output_filename = "processed_video.mp4"
+        cap = cv2.VideoCapture(filename)
+        frame_width = int(cap.get(3))
+        frame_height = int(cap.get(4))
+        out = cv2.VideoWriter(output_filename, cv2.VideoWriter_fourcc(*'h264'), 30, (frame_width, frame_height))
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = self.pose.process(frame_rgb)
+            processed_frame = self.process_frame(frame, results)
+            out.write(processed_frame)
+        cap.release()
+        out.release()
+
+        # Remove the temporary file
+        os.remove(filename)
+
+        # Return the path to the processed video file
+        return output_filename
+    
+    def process_frame(self, frame, results):
+        # Process the frame using the `process` function
+        processed_frame = self.process(frame)
+        return processed_frame
+    
+    def process(self, image):
+      
+        # Pose detection model
+        image.flags.writeable = False
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        results = pose.process(image)
+
+        # Draw the hand annotations on the image.
+        image.flags.writeable = True
+        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+        self.draw_landmarks(image, results) 
+        
+        # Prediction logic
+        keypoints = self.extract_keypoints(results)        
+        self.sequence.append(keypoints.astype('float32',casting='same_kind'))      
+        self.sequence = self.sequence[-self.sequence_length:]
+        
+        if len(self.sequence) == self.sequence_length:
+            res = self.model.predict(np.expand_dims(self.sequence, axis=0), verbose=0)[0]
+            
+            self.current_action = self.actions[np.argmax(res)]
+            confidence = np.max(res)
+            print("confidence", confidence)  # Debug print statement
+            print("current action" , self.current_action)
+            
+            # Erase current action variable if no probability is above threshold
+            if confidence < self.threshold:
+                self.current_action = ''
+                
+                
+            print("current action" , self.current_action)
+
+
+            # Viz probabilities
+            image = self.prob_viz(res, image)
+            
+            # Count reps
+            
+            landmarks = results.pose_landmarks.landmark
+            self.count_reps(image, landmarks, mp_pose)
+            
+
+            # Display graphical information
+            cv2.rectangle(image, (0,0), (640, 40), self.colors[np.argmax(res)], -1)
+            cv2.putText(image, 'curl ' + str(self.curl_counter), (3,30), 
+                           cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
+            cv2.putText(image, 'press ' + str(self.press_counter), (240,30), 
+                           cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
+            cv2.putText(image, 'squat ' + str(self.squat_counter), (490,30), 
+                           cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
+          
+        return image
+
+    def draw_landmarks(self, image, results):
+        mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,
+                                  mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2), 
+                                  mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2))
+        return image
+
+    def extract_keypoints(self, results):
+        pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)
+        return pose
+
+    def count_reps(self, image, landmarks, mp_pose):
+        """
+        Counts repetitions of each exercise. Global count and stage (i.e., state) variables are updated within this function.
+
+        """
+
+        if self.current_action == 'curl':
+            # Get coords
+            shoulder = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'SHOULDER')
+            elbow = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'ELBOW')
+            wrist = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'WRIST')
+
+            # calculate elbow angle
+            angle = self.calculate_angle(shoulder, elbow, wrist)
+
+            # curl counter logic
+            print("Curl Angle:", angle)  # Debug print statement
+            if angle < 30:
+                self.curl_stage = "up"
+            if angle > 140 and self.curl_stage == 'up':
+                self.curl_stage = "down"
+                self.curl_counter += 1
+                print("count:",self.curl_counter)
+            self.press_stage = None
+            self.squat_stage = None
+
+            # Viz joint angle
+            self.viz_joint_angle(image, angle, elbow)
+
+        elif self.current_action == 'press':
+            # Get coords
+            shoulder = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'SHOULDER')
+            elbow = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'ELBOW')
+            wrist = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'WRIST')
+
+            # Calculate elbow angle
+            elbow_angle = self.calculate_angle(shoulder, elbow, wrist)
+            print(shoulder, elbow, wrist)
+            # Compute distances between joints
+            shoulder2elbow_dist = abs(math.dist(shoulder, elbow))
+            shoulder2wrist_dist = abs(math.dist(shoulder, wrist))
+
+            # Press counter logic
+            print("Press Angle:", elbow_angle)  # Debug print statement
+            print("Shoulder to Elbow Distance:", shoulder2elbow_dist)  # Debug print statement
+            print("Shoulder to Wrist Distance:", shoulder2wrist_dist)  # Debug print statement
+            if (elbow_angle > 130) and (shoulder2elbow_dist < shoulder2wrist_dist):
+                self.press_stage = "up"
+            if (elbow_angle < 50) and (shoulder2elbow_dist > shoulder2wrist_dist) and (self.press_stage == 'up'):
+                self.press_stage = 'down'
+                self.press_counter += 1
+                
+                
+                print("count:",self.press_counter)
+            self.curl_stage = None
+            self.squat_stage = None
+
+            # Viz joint angle
+            self.viz_joint_angle(image, elbow_angle, elbow)
+
+        elif self.current_action == 'squat':
+            # Get coords
+            # left side
+            left_shoulder = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'SHOULDER')
+            left_hip = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'HIP')
+            left_knee = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'KNEE')
+            left_ankle = self.get_coordinates(landmarks, mp_pose, 'LEFT', 'ANKLE')
+            # right side
+            right_shoulder = self.get_coordinates(landmarks, mp_pose, 'RIGHT', 'SHOULDER')
+            right_hip = self.get_coordinates(landmarks, mp_pose, 'RIGHT', 'HIP')
+            right_knee = self.get_coordinates(landmarks, mp_pose, 'RIGHT', 'KNEE')
+            right_ankle = self.get_coordinates(landmarks, mp_pose, 'RIGHT', 'ANKLE')
+
+            # Calculate knee angles
+            left_knee_angle = self.calculate_angle(left_hip, left_knee, left_ankle)
+            right_knee_angle = self.calculate_angle(right_hip, right_knee, right_ankle)
+
+            # Calculate hip angles
+            left_hip_angle = self.calculate_angle(left_shoulder, left_hip, left_knee)
+            right_hip_angle = self.calculate_angle(right_shoulder, right_hip, right_knee)
+
+            # Squat counter logic
+            thr = 165
+            print("Left Knee Angle:", left_knee_angle)  # Debug print statement
+            print("Right Knee Angle:", right_knee_angle)  # Debug print statement
+            print("Left Hip Angle:", left_hip_angle)  # Debug print statement
+            print("Right Hip Angle:", right_hip_angle)  # Debug print statement
+            if (left_knee_angle < thr) and (right_knee_angle < thr) and (left_hip_angle < thr) and (
+                    right_hip_angle < thr):
+                self.squat_stage = "down"
+            if (left_knee_angle > thr) and (right_knee_angle > thr) and (left_hip_angle > thr) and (
+                    right_hip_angle > thr) and (self.squat_stage == 'down'):
+                self.squat_stage = 'up'
+                self.squat_counter += 1
+                print("count:",self.squat_counter)
+            self.curl_stage = None
+            self.press_stage = None
+
+            # Viz joint angles
+            self.viz_joint_angle(image, left_knee_angle, left_knee)
+            self.viz_joint_angle(image, left_hip_angle, left_hip)
+
+        else:
+            pass
+        return
+
+    def prob_viz(self, res, input_frame):
+        """
+        This function displays the model prediction probability distribution over the set of exercise classes
+        as a horizontal bar graph
+        
+        """
+        output_frame = input_frame.copy()
+        for num, prob in enumerate(res):        
+            cv2.rectangle(output_frame, (0,60+num*40), (int(prob*100), 90+num*40), self.colors[num], -1)
+            cv2.putText(output_frame, self.actions[num], (0, 85+num*40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.LINE_AA)
+            
+        return output_frame
+
+    def get_coordinates(self, landmarks, mp_pose, side, part):
+        
+        
+        coord = getattr(mp_pose.PoseLandmark,side.upper()+"_"+part.upper())
+        x_coord_val = landmarks[coord.value].x
+        y_coord_val = landmarks[coord.value].y
+        return [x_coord_val, y_coord_val] 
+    
+    
+   
+       
+
+
+
+    def calculate_angle(self, a, b, c):
+        a = np.array(a) 
+        b = np.array(b) 
+        c = np.array(c)
+        radians = math.atan2(c[1]-b[1], c[0]-b[0]) - math.atan2(a[1]-b[1], a[0]-b[0])
+        angle = np.abs(radians*180.0/np.pi)
+        if angle > 180.0:
+            angle = 360 - angle
+        return angle
+
+    def viz_joint_angle(self, image, angle, joint):
+        cv2.putText(image, str(round(angle, 2)), 
+                    tuple(np.multiply(joint, [640, 480]).astype(int)), 
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,255,255), 2, cv2.LINE_AA)
+
+# Define Streamlit app
+def main():
+    st.title("Real-time Exercise Detection")
+    video_file = st.file_uploader("Upload a video file", type=["mp4", "avi"])
+    if video_file is not None:
+        video_processor = VideoProcessor()
+        
+        output_video = video_processor.process_video(video_file)
+        
+        
+        video_file = open(output_video, 'rb')
+        video_bytes = video_file.read()
+        st.video(video_bytes)
+
+if __name__ == "__main__":
+    main()

Exercise_Recognition_AI-main/models/LSTM.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6778664cec93d5e917b064af44e03dfb4344c3a779d453106d68c1d3ea00e560
+size 9069616

Exercise_Recognition_AI-main/models/LSTM_Attention.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2395d5eb371bb8221e2cacb7c98dbc336de6775bd2607747f4e1f72d0fa4e915
+size 104036816

Exercise_Recognition_AI-main/test.py

@@ -0,0 +1,3 @@
+import streamlit as st
+
+st.video("C:/Users/amite/Downloads/Exercise_Recognition_AI-main/Exercise_Recognition_AI-main/processed_video.mp4")

Exercise_Recognition_AI-main/tests/feature_engineering.ipynb

@@ -0,0 +1,295 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 242,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import cv2\n",
+    "import numpy as np\n",
+    "import os\n",
+    "from matplotlib import pyplot as plt\n",
+    "import time\n",
+    "import mediapipe as mp\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 243,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Pre-trained pose estimation model from Google Mediapipe\n",
+    "mp_pose = mp.solutions.pose\n",
+    "\n",
+    "# Supported Mediapipe visualization tools\n",
+    "mp_drawing = mp.solutions.drawing_utils"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 244,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def mediapipe_detection(image, model):\n",
+    "    \"\"\"\n",
+    "    This function detects human pose estimation keypoints from webcam footage\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB\n",
+    "    image.flags.writeable = False                  # Image is no longer writeable\n",
+    "    results = model.process(image)                 # Make prediction\n",
+    "    image.flags.writeable = True                   # Image is now writeable \n",
+    "    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR\n",
+    "    return image, results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 245,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def draw_landmarks(image, results):\n",
+    "    \"\"\"\n",
+    "    This function draws keypoints and landmarks detected by the human pose estimation model\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,\n",
+    "                                mp_drawing.DrawingSpec(color=(245,117,66), thickness=2, circle_radius=2), \n",
+    "                                mp_drawing.DrawingSpec(color=(245,66,230), thickness=2, circle_radius=2) \n",
+    "                                 )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 246,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def draw_detection(image, results):\n",
+    "\n",
+    "    h, w, c = image.shape\n",
+    "    cx_min = w\n",
+    "    cy_min = h\n",
+    "    cx_max = cy_max = 0\n",
+    "    center = [w//2, h//2]\n",
+    "    try:\n",
+    "        for id, lm in enumerate(results.pose_landmarks.landmark):\n",
+    "            cx, cy = int(lm.x * w), int(lm.y * h)\n",
+    "            if cx < cx_min:\n",
+    "                cx_min = cx\n",
+    "            if cy < cy_min:\n",
+    "                cy_min = cy\n",
+    "            if cx > cx_max:\n",
+    "                cx_max = cx\n",
+    "            if cy > cy_max:\n",
+    "                cy_max = cy\n",
+    "                \n",
+    "        boxW, boxH = cx_max - cx_min, cy_max - cy_min\n",
+    "        \n",
+    "        # center\n",
+    "        cx, cy = cx_min + (boxW // 2), \\\n",
+    "                 cy_min + (boxH // 2)      \n",
+    "        center = [cx, cy]\n",
+    "                \n",
+    "        cv2.rectangle(\n",
+    "            image, (cx_min, cy_min), (cx_max, cy_max), (255, 255, 0), 2\n",
+    "        )\n",
+    "    except:\n",
+    "        pass\n",
+    "    \n",
+    "    return [[cx_min, cy_min], [cx_max, cy_max]], center"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 247,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def normalize(image, results, bounding_box, landmark_names):\n",
+    "    h, w, c = image.shape\n",
+    "    if results.pose_landmarks:\n",
+    "        xy = {}\n",
+    "        xy_norm = {}\n",
+    "        i = 0\n",
+    "        for res in results.pose_landmarks.landmark:\n",
+    "            x = res.x * w\n",
+    "            y = res.y * h\n",
+    "            \n",
+    "            x_norm = (x - bounding_box[0][0]) / (bounding_box[1][0] - bounding_box[0][0])\n",
+    "            y_norm = (y - bounding_box[0][1]) / (bounding_box[1][1] - bounding_box[0][1])\n",
+    "            \n",
+    "            # xy_norm.append([x_norm, y_norm])\n",
+    "            \n",
+    "            xy_norm[landmark_names[i]] = [x_norm, y_norm]\n",
+    "            i += 1\n",
+    "    else:\n",
+    "        # xy_norm = np.zeros([0,0] * 33)\n",
+    "        \n",
+    "        # xy = {landmark_names: [0,0]}\n",
+    "        # xy_norm = {landmark_names: [0,0]}\n",
+    "        \n",
+    "        xy_norm = dict(zip(landmark_names, [0,0] * 33))\n",
+    "    \n",
+    "    return xy_norm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 248,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_coordinates(landmarks, mp_pose, side, joint):\n",
+    "    \"\"\"\n",
+    "    Retrieves x and y coordinates of a particular keypoint from the pose estimation model\n",
+    "         \n",
+    "     Args:\n",
+    "         landmarks: processed keypoints from the pose estimation model\n",
+    "         mp_pose: Mediapipe pose estimation model\n",
+    "         side: 'left' or 'right'. Denotes the side of the body of the landmark of interest.\n",
+    "         joint: 'shoulder', 'elbow', 'wrist', 'hip', 'knee', or 'ankle'. Denotes which body joint is associated with the landmark of interest.\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    coord = getattr(mp_pose.PoseLandmark,side.upper()+\"_\"+joint.upper())\n",
+    "    x_coord_val = landmarks[coord.value].x\n",
+    "    y_coord_val = landmarks[coord.value].y\n",
+    "    return [x_coord_val, y_coord_val]    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 249,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def viz_coords(image, norm_coords, landmarks, mp_pose, side, joint):\n",
+    "    \"\"\"\n",
+    "    Displays the joint angle value near the joint within the image frame\n",
+    "    \n",
+    "    \"\"\"\n",
+    "    try:\n",
+    "          point = side.upper()+\"_\"+joint.upper()\n",
+    "          norm_coords = norm_coords[point]\n",
+    "          joint = get_coordinates(landmarks, mp_pose, side, joint)\n",
+    "          \n",
+    "          coords = [ '%.2f' % elem for elem in joint ]\n",
+    "          coords = ' '.join(str(coords))\n",
+    "          norm_coords = [ '%.2f' % elem for elem in norm_coords ]\n",
+    "          norm_coords = ' '.join(str(norm_coords))\n",
+    "          cv2.putText(image, coords, \n",
+    "                         tuple(np.multiply(joint, [640, 480]).astype(int)), \n",
+    "                         cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, cv2.LINE_AA\n",
+    "                              )\n",
+    "          cv2.putText(image, norm_coords, \n",
+    "                         tuple(np.multiply(joint, [640, 480]).astype(int) + 20), \n",
+    "                         cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,255), 2, cv2.LINE_AA\n",
+    "                              )\n",
+    "    except:\n",
+    "         pass\n",
+    "    return"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 250,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cap = cv2.VideoCapture(0) # camera object\n",
+    "HEIGHT = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # webcam video frame height\n",
+    "WIDTH = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # webcam video frame width\n",
+    "FPS = int(cap.get(cv2.CAP_PROP_FPS)) # webcam video fram rate \n",
+    "\n",
+    "landmark_names = dir(mp_pose.PoseLandmark)[:-4]\n",
+    "\n",
+    "# Set and test mediapipe model using webcam\n",
+    "with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5, enable_segmentation=True) as pose:\n",
+    "    while cap.isOpened():\n",
+    "\n",
+    "        # Read feed\n",
+    "        ret, frame = cap.read()\n",
+    "      \n",
+    "        # Make detection\n",
+    "        image, results = mediapipe_detection(frame, pose)\n",
+    "        \n",
+    "        # Extract landmarks\n",
+    "        try:\n",
+    "            landmarks = results.pose_landmarks.landmark\n",
+    "        except:\n",
+    "            pass\n",
+    "        \n",
+    "        # draw bounding box\n",
+    "        bounding_box, box_center = draw_detection(image, results)\n",
+    "           \n",
+    "        # Render detections\n",
+    "        draw_landmarks(image, results)  \n",
+    "        \n",
+    "        # normalize coordinates\n",
+    "        xy_norm = normalize(image, results, bounding_box, landmark_names) \n",
+    "        viz_coords(image, xy_norm, landmarks, mp_pose, 'left', 'wrist')   \n",
+    "        viz_coords(image, xy_norm, landmarks, mp_pose, 'right', 'wrist')         \n",
+    "        \n",
+    "        # Display frame on screen\n",
+    "        cv2.imshow('OpenCV Feed', image)\n",
+    "        \n",
+    "        # Draw segmentation on the image.\n",
+    "        # To improve segmentation around boundaries, consider applying a joint\n",
+    "        # bilateral filter to \"results.segmentation_mask\" with \"image\".\n",
+    "        # tightness = 0.3 # Probability threshold in [0, 1] that says how \"tight\" to make the segmentation. Greater value => tighter.\n",
+    "        # condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > tightness\n",
+    "        # bg_image = np.zeros(image.shape, dtype=np.uint8)\n",
+    "        # bg_image[:] = (192, 192, 192) # gray\n",
+    "        # image = np.where(condition, image, bg_image)\n",
+    "        \n",
+    "        # Exit / break out logic\n",
+    "        if cv2.waitKey(10) & 0xFF == ord('q'):\n",
+    "            break\n",
+    "\n",
+    "    cap.release()\n",
+    "    cv2.destroyAllWindows()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 251,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cap.release()\n",
+    "cv2.destroyAllWindows()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.8.13 ('AItrainer')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.13"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "80aa1d3f3a8cfb37a38c47373cc49a39149184c5fa770d709389b1b8782c1d85"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}