Gait Analysis Output Video frame is very slow. How can I optimize the code?

[Aastha29P]

When I ran my poseDetectVideo.py, the output is very slow. The video moves from one frame to another in 7 seconds. I haven't used the 3rdparty folder in my code. Maybe that is required to optimise the code. Also, where can I use the cmake and 3rdparty features in my code.
Currently, I have poseDetectVideo.py, getModels.sh, data folder (which contains the video file) and models folder (provided in the above messages). I am running these in vs code.
Can anyone guide me how can I optimise the code and how can I leverage cmake and 3rdparty to my code?
poseDetectVideo.py

import the necessary packages
import time

import cv2
import imutils
import numpy as np
from imutils.video import FileVideoStream

fvs = FileVideoStream('data/cam1.mp4', queue_size=1024).start()
time.sleep(1.0)

kernelSize = 7
backgroundHistory = 15

openposeProtoFile = "models/pose/coco/pose_deploy_linevec.prototxt"
openposeWeightsFile = "models/pose/coco/pose_iter_440000.caffemodel"
nPoints = 18

COCO Output Format
keypointsMapping = ['Nose', 'Neck', 'R-Sho', 'R-Elb', 'R-Wr', 'L-Sho', 'L-Elb', 'L-Wr', 'R-Hip', 'R-Knee', 'R-Ank',
'L-Hip', 'L-Knee', 'L-Ank', 'R-Eye', 'L-Eye', 'R-Ear', 'L-Ear']

POSE_PAIRS = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7],
[1, 8], [8, 9], [9, 10], [1, 11], [11, 12], [12, 13],
[1, 0], [0, 14], [14, 16], [0, 15], [15, 17],
[2, 17], [5, 16]]

index of pafs correspoding to the POSE_PAIRS
e.g for POSE_PAIR(1,2), the PAFs are located at indices (31,32) of output, Similarly, (1,5) -> (39,40) and so on.
mapIdx = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44],
[19, 20], [21, 22], [23, 24], [25, 26], [27, 28], [29, 30],
[47, 48], [49, 50], [53, 54], [51, 52], [55, 56],
[37, 38], [45, 46]]

colors = [[0, 100, 255], [0, 100, 255], [0, 255, 255], [0, 100, 255], [0, 255, 255], [0, 100, 255],
[0, 255, 0], [255, 200, 100], [255, 0, 255], [0, 255, 0], [255, 200, 100], [255, 0, 255],
[0, 0, 255], [255, 0, 0], [200, 200, 0], [255, 0, 0], [200, 200, 0], [0, 0, 0]]

def getKeypoints(prob_map, thres=0.1):
map_smooth = cv2.GaussianBlur(prob_map, (3, 3), 0, 0)

map_mask = np.uint8(map_smooth > thres)
keypoints_array = []

find the blobs

contours, _ = cv2.findContours(map_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

for each blob find the maxima

for cnt in contours:
blob_mask = np.zeros(map_mask.shape)
blob_mask = cv2.fillConvexPoly(blob_mask, cnt, 1)
masked_prob_map = map_smooth * blob_mask
_, max_val, _, max_loc = cv2.minMaxLoc(masked_prob_map)
keypoints_array.append(max_loc + (prob_map[max_loc[1], max_loc[0]],))

return keypoints_array
Find valid connections between the different joints of a all persons present
def getValidPairs(generated_output):
validpairs = []
invalidpairs = []
n_interp_samples = 10
paf_score_th = 0.1
conf_th = 0.7

loop for every POSE_PAIR

for k in range(len(mapIdx)):

A->B constitute a limb

pafA = generated_output[0, mapIdx[k][0], :, :]
pafB = generated_output[0, mapIdx[k][1], :, :]
pafA = cv2.resize(pafA, (frameWidth, frameHeight))
pafB = cv2.resize(pafB, (frameWidth, frameHeight))

# Find the keypoints for the first and second limb
candA = detected_keypoints[POSE_PAIRS[k][0]]
candB = detected_keypoints[POSE_PAIRS[k][1]]
nA = len(candA)
nB = len(candB)

# If keypoints for the joint-pair is detected
# check every joint in candA with every joint in candB
# Calculate the distance vector between the two joints
# Find the PAF values at a set of interpolated points between the joints
# Use the above formula to compute a score to mark the connection valid

if nA != 0 and nB != 0:
    valid_pair = np.zeros((0, 3))
    for i in range(nA):
        max_j = -1
        max_score = -1
        found = 0
        for j in range(nB):
            # Find d_ij
            d_ij = np.subtract(candB[j][:2], candA[i][:2])
            norm = np.linalg.norm(d_ij)
            if norm:
                d_ij = d_ij / norm
            else:
                continue
            # Find p(u)
            interp_coord = list(zip(np.linspace(candA[i][0], candB[j][0], num=n_interp_samples),
                                    np.linspace(candA[i][1], candB[j][1], num=n_interp_samples)))
            # Find L(p(u))
            paf_interp = []
            for k in range(len(interp_coord)):
                paf_interp.append([pafA[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))],
                                   pafB[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))]])
            # Find E
            paf_scores = np.dot(paf_interp, d_ij)
            avg_paf_score = sum(paf_scores) / len(paf_scores)

            # Check if the connection is valid
            # If the fraction of interpolated vectors aligned with PAF is higher then threshold -> Valid Pair
            if (len(np.where(paf_scores > paf_score_th)[0]) / n_interp_samples) > conf_th:
                if avg_paf_score > max_score:
                    max_j = j
                    max_score = avg_paf_score
                    found = 1
        # Append the connection to the list
        if found:
            valid_pair = np.append(valid_pair, [[candA[i][3], candB[max_j][3], max_score]], axis=0)

    # Append the detected connections to the global list
    validpairs.append(valid_pair)
else:  # If no keypoints are detected
    invalidpairs.append(k)
    validpairs.append([])

return validpairs, invalidpairs
This function creates a list of keypoints belonging to each person
For each detected valid pair, it assigns the joint(s) to a person
def getPersonwiseKeypoints(validpairs, invalidpairs):

the last number in each row is the overall score

personwise_keypoints = -1 * np.ones((0, 19))

for k in range(len(mapIdx)):
if k not in invalidpairs:
partAs = validpairs[k][:, 0]
partBs = validpairs[k][:, 1]
indexA, indexB = np.array(POSE_PAIRS[k])

    for i in range(len(validpairs[k])):
        found = 0
        person_idx = -1
        for j in range(len(personwise_keypoints)):
            if personwise_keypoints[j][indexA] == partAs[i]:
                person_idx = j
                found = 1
                break

        if found:
            personwise_keypoints[person_idx][indexB] = partBs[i]
            personwise_keypoints[person_idx][-1] += keypoints_list[partBs[i].astype(int), 2] + validpairs[k][i][
                2]

        # if find no partA in the subset, create a new subset
        elif not found and k < 17:
            row = -1 * np.ones(19)
            row[indexA] = partAs[i]
            row[indexB] = partBs[i]
            # add the keypoint_scores for the two keypoints and the paf_score
            row[-1] = sum(keypoints_list[validpairs[k][i, :2].astype(int), 2]) + validpairs[k][i][2]
            personwise_keypoints = np.vstack([personwise_keypoints, row])

return personwise_keypoints
fgbg = cv2.createBackgroundSubtractorMOG2(history=backgroundHistory, detectShadows=True)
kernel = np.ones((kernelSize, kernelSize), np.uint8)

while fvs.more():
frame = fvs.read()
frame = imutils.resize(frame, width=960)

frameClone = frame.copy()

frameWidth = frame.shape[1]
frameHeight = frame.shape[0]

Fix the input Height and get the width according to the Aspect Ratio

inHeight = 368
inWidth = int((inHeight / frameHeight) * frameWidth)
inpBlob = cv2.dnn.blobFromImage(frame, 1.0 / 255, (inWidth, inHeight), (0, 0, 0), swapRB=False, crop=False)

net = cv2.dnn.readNetFromCaffe(openposeProtoFile, openposeWeightsFile)

net.setInput(inpBlob)
output = net.forward()

Applying background subtraction on the capture frame

frame = fgbg.apply(frame)

detected_keypoints = []
keypoints_list = np.zeros((0, 3))
keypoint_id = 0
threshold = 0.1

for part in range(nPoints):
probMap = output[0, part, :, :]
probMap = cv2.resize(probMap, (frame.shape[1], frame.shape[0]))
keypoints = getKeypoints(probMap, threshold)

keypoints_with_id = []
for i in range(len(keypoints)):
    keypoints_with_id.append(keypoints[i] + (keypoint_id,))
    keypoints_list = np.vstack([keypoints_list, keypoints[i]])
    keypoint_id += 1

detected_keypoints.append(keypoints_with_id)

for i in range(nPoints):

for j in range(len(detected_keypoints[i])):

cv2.circle(frame, detected_keypoints[i][j][0:2], 5, colors[i], -1, cv2.LINE_AA)

cv2.imshow("Keypoints", frame)

valid_pairs, invalid_pairs = getValidPairs(output)
personwiseKeypoints = getPersonwiseKeypoints(valid_pairs, invalid_pairs)

for i in range(17):
for n in range(len(personwiseKeypoints)):
index = personwiseKeypoints[n][np.array(POSE_PAIRS[i])]
if -1 in index:
continue
B = np.int32(keypoints_list[index.astype(int), 0])
A = np.int32(keypoints_list[index.astype(int), 1])
cv2.line(frame, (B[0], A[0]), (B[1], A[1]), colors[i], 2, cv2.LINE_AA)

frame = cv2.addWeighted(frameClone, 0.5, frame, 0.5, 0.0)

cv2.imshow("Frame", frame)
k = cv2.waitKey(50) & 0xff
if k == 27:
break
do a bit of cleanup
cv2.destroyAllWindows()
fvs.stop()

getModels.sh

------------------------- BODY, FACE AND HAND MODELS -------------------------
Setting paths to the downloaded models
OPENPOSE_Models="models/"
POSE_FOLDER="models/pose/"
FACE_FOLDER="models/face/"
HAND_FOLDER="models/hand/"

------------------------- POSE MODELS -------------------------
Body (COCO)
COCO_MODEL="${OPENPOSE_Models}pose/coco/pose_iter_440000.caffemodel"

Body (MPI)
MPI_MODEL="${OPENPOSE_Models}pose/mpi/pose_iter_160000.caffemodel"

------------------------- FACE MODELS -------------------------
Face
FACE_MODEL="${OPENPOSE_Models}face/pose_iter_116000.caffemodel"

------------------------- HAND MODELS -------------------------
Hand
HAND_MODEL="${OPENPOSE_Models}hand/pose_iter_102000.caffemodel"

I have pasted the 3rdParty folder to my directory but I don't know how to use that and cmake in my code.
Your help is appreciated. Thanks!