This time, we will specify the tracking target for the video with a mouse click and try to track the object in the 3D space in real time. Compared to template matching, it uses an algorithm that uses features, which requires less calculation and can be tracked even when rotated.
OpenCV OpenCV (Open Source Computer Vision Library) is a collection of BSD-licensed video / image processing libraries. There are many algorithms for image filtering, template matching, object recognition, video analysis, machine learning, and more.
■ Example of motion tracking using OpenCV (OpenCV Google Summer of Code 2015) https://www.youtube.com/watch?v=OUbUFn71S4s
■ Click here for installation and easy usage Install OpenCV 3 (core + contrib) in Python 3 environment & Difference between OpenCV 2 and OpenCV 3 & simple operation check
■ Click here for still image filtering Try edge detection with OpenCV Perform various filters with OpenCV (gradient, high-pass, Laplacian, Gaussian) Extract feature points with OpenCV (AgastFeature, FAST, GFTT, MSER, AKAZE, BRISK, KAZE, ORB, SimpleBlob)
■ Click here for processing video files Try converting videos in real time with OpenCV Try converting webcam / camcorder video in real time with OpenCV Draw optical flow in real time with OpenCV (Shi-Tomasi method, Lucas-Kanade method)
This time, create a program with the following functions.
The pause function is quite important when specifying feature points for a video with a mouse click. Pause the video with the "s" key and slowly specify the feature points.
Operating environment
Video data I used the sample video that comes with OpenCV. OpenCV\opencv\sources\samples\data\768x576.avi
motion.py
import cv2
import numpy as np
#Esc key
ESC_KEY = 0x1b
#s key
S_KEY = 0x73
#r key
R_KEY = 0x72
#Maximum number of feature points
MAX_FEATURE_NUM = 500
#Termination condition of iterative algorithm
CRITERIA = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)
#Interval (1000/frame rate)
INTERVAL = 30
#Video data
VIDEO_DATA = '768x576.avi'
class Motion:
#constructor
def __init__(self):
#Display window
cv2.namedWindow("motion")
#Mouse event callback registration
cv2.setMouseCallback("motion", self.onMouse)
#Video
self.video = cv2.VideoCapture(VIDEO_DATA)
#interval
self.interval = INTERVAL
#Current frame (color)
self.frame = None
#Current frame (gray)
self.gray_next = None
#Last frame (gray)
self.gray_prev = None
#Feature point
self.features = None
#Feature point status
self.status = None
#Main loop
def run(self):
#Processing of the first frame
end_flag, self.frame = self.video.read()
self.gray_prev = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
while end_flag:
#Convert to grayscale
self.gray_next = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
#Calculate Optical Flow when feature points are registered
if self.features is not None:
#Optical flow calculation
features_prev = self.features
self.features, self.status, err = cv2.calcOpticalFlowPyrLK( \
self.gray_prev, \
self.gray_next, \
features_prev, \
None, \
winSize = (10, 10), \
maxLevel = 3, \
criteria = CRITERIA, \
flags = 0)
#Leave only valid feature points
self.refreshFeatures()
#Draw valid feature points on the frame
if self.features is not None:
for feature in self.features:
cv2.circle(self.frame, (feature[0][0], feature[0][1]), 4, (15, 241, 255), -1, 8, 0)
#display
cv2.imshow("motion", self.frame)
#Preparing for the next loop
self.gray_prev = self.gray_next
end_flag, self.frame = self.video.read()
if end_flag:
self.gray_next = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
#interval
key = cv2.waitKey(self.interval)
# "Esc"Press the key to finish
if key == ESC_KEY:
break
# "s"Pause by pressing a key
elif key == S_KEY:
self.interval = 0
elif key == R_KEY:
self.interval = INTERVAL
#End processing
cv2.destroyAllWindows()
self.video.release()
#Specify feature points with a mouse click
#If there is an existing feature point in the clicked neighborhood, delete the existing feature point
#If there is no existing feature point in the clicked neighborhood, add a new feature point
def onMouse(self, event, x, y, flags, param):
#Other than left click
if event != cv2.EVENT_LBUTTONDOWN:
return
#Addition of first feature point
if self.features is None:
self.addFeature(x, y)
return
#Search radius (pixel)
radius = 5
#Search for existing feature points in the vicinity
index = self.getFeatureIndex(x, y, radius)
#Since there is an existing feature point near the clicked area, delete the existing feature point.
if index >= 0:
self.features = np.delete(self.features, index, 0)
self.status = np.delete(self.status, index, 0)
#Since there is no existing feature point near the clicked area, a new feature point is added.
else:
self.addFeature(x, y)
return
#Get one index of existing feature points within the specified radius
#If there are no feature points within the specified radius index= -Reply 1
def getFeatureIndex(self, x, y, radius):
index = -1
#No feature points are registered
if self.features is None:
return index
max_r2 = radius ** 2
index = 0
for point in self.features:
dx = x - point[0][0]
dy = y - point[0][1]
r2 = dx ** 2 + dy ** 2
if r2 <= max_r2:
#This feature point is within the specified radius
return index
else:
#This feature point is outside the specified radius
index += 1
#All feature points are outside the specified radius
return -1
#Add new feature points
def addFeature(self, x, y):
#Feature points are not registered
if self.features is None:
#Create ndarray and register the coordinates of feature points
self.features = np.array([[[x, y]]], np.float32)
self.status = np.array([1])
#High accuracy of feature points
cv2.cornerSubPix(self.gray_next, self.features, (10, 10), (-1, -1), CRITERIA)
#Exceeds the maximum number of feature points registered
elif len(self.features) >= MAX_FEATURE_NUM:
print("max feature num over: " + str(MAX_FEATURE_NUM))
#Additional feature points registered
else:
#Add feature point coordinates to the end of an existing ndarray
self.features = np.append(self.features, [[[x, y]]], axis = 0).astype(np.float32)
self.status = np.append(self.status, 1)
#High accuracy of feature points
cv2.cornerSubPix(self.gray_next, self.features, (10, 10), (-1, -1), CRITERIA)
#Leave only valid feature points
def refreshFeatures(self):
#Feature points are not registered
if self.features is None:
return
#Check all status
i = 0
while i < len(self.features):
#Cannot be recognized as a feature point
if self.status[i] == 0:
#Remove from existing ndarray
self.features = np.delete(self.features, i, 0)
self.status = np.delete(self.status, i, 0)
i -= 1
i += 1
if __name__ == '__main__':
Motion().run()
The program used the OpenCV sample video (768x576.avi), but it doesn't always look the same on the same screen.
Any video is OK. Change the VIDEO_DATA
part of the program to your favorite video.
For example, let's use Uchimura, who plays the horizontal bar, in the Rio de Janeiro Olympic 2016 Gymnastics Individual All-Around Final as an example of the execution results. In this video, Uchimura, who plays the horizontal bar, is also moving, but the background frame is also moving up and down according to Uchimura. Use the "s" key to pause the video and set the point to follow Uchimura's socks. The yellow dots represent the feature points. When you play the video, you can see that the yellow dots are tracking Uchimura's socks. (Because Uchimura's toes are neatly aligned, it can be said that he can continue to recognize it as the same feature point.)
When you actually run the program, you can see that this program (simply using cv2.calcOpticalFlowPyrLK ()) has three problems.
In both cases, the amount of calculation increases, but there are ways to deal with it. I will try to make a compatible version soon.
Recommended Posts