Machine Learning - SVD code

Video Backgroud Removal

Library

import os
import scipy
import numpy as np

import moviepy.editor as mpy  # install moviepy package if not installed 
from moviepy.video.io.bindings import mplfig_to_npimage

%matplotlib inline
import matplotlib.pyplot as plt
from sklearn import decomposition

from PIL import Image

Load the video

video = mpy.VideoFileClip("Video_003.avi")
#original surveillance video 
video.subclip(0,50).ipython_display(width=300)
video.duration, video.size

Helper functions

def rgb2gray(rgb):  # Grayscale as single channels of multichannel color images
    return np.dot(rgb[...,:3], [0.299, 0.587, 0.114])  # Dot product of two arrays

def check_frames(clip, frames): 
    num_frames = clip.fps * clip.duration
    desired_dur = frames / clip.fps 
    assert desired_dur <= clip.duration, 'too many frames requested! must be less than {}'.format(num_frames)

def video_to_matrix(clip, frames=5, scale = (160,120)):
  return np.vstack([np.array(Image.fromarray(rgb2gray(clip.get_frame(i/float(frames))).astype(np.uint8)).resize(scale)).flatten() for i in range(frames * int(clip.duration))]).T

Format the data

orig_img_height = video.size[0] # 320
orig_img_width = video.size[1] # 240

scale = (160, 120)
scale_num = 50  # Adjust scale to change resolution of image, scale to percent (100 means no scaling) 
dims = (int(orig_img_width * (scale_num/100)), int(orig_img_height * (scale_num/100)))

video_mat = video_to_matrix(video, frames=794, scale=scale)
print('dims: {} \nvideo matrix shape {}'.format(dims, video_mat.shape))

Naive background removal

plt.figure(figsize=(10, 10))
plt.imshow(np.sort(video_mat), aspect=.02, cmap='gray')

mode_col = int(video_mat.shape[1]/2)
sorted_mat = np.sort(video_mat)
background = sorted_mat[:, mode_col]

plt.figure(figsize=(10, 10))
plt.imshow(background.reshape(dims), cmap='gray')

from scipy import stats

random_columns = np.random.randint(0, video_mat.shape[1], 50)
background = stats.mode(video_mat[:, random_columns], axis=1).mode
#np.save('data/background', background)
background.shape

plt.figure(figsize=(10, 10))
plt.imshow(background.reshape(dims), cmap='gray');

Extract people

people = video_mat - background.reshape(-1, 1)
people.shape

def plot_two_images(img1, img2, cmap='gray'): 
    
    plt.figure(figsize=(12, 12))
    plt.subplot(1, 2, 1);  plt.title("Original Image")
    plt.imshow(img1, cmap='gray')

    plt.subplot(1, 2, 2); plt.title("Background Removed")
    plt.imshow(img2, cmap='gray')
    plt.show()
    
 plot_two_images(video_mat[:,20].reshape(dims),
                people[:, 20].reshape(dims))

noise 제거

people[(people < 25) | (people > 210)] = 0
plot_two_images(video_mat[:,20].reshape(dims), people[:, 20].reshape(dims))

SVD

U, s, V = np.linalg.svd(video_mat, full_matrices=False)

np.save("video/U.npy", U)
np.save("video/s.npy", s)
np.save("video/V.npy", V)

U = np.load("video/U.npy")
s = np.load("video/s.npy")
V = np.load("video/V.npy")

MAKE A VIDEO

people_bg_removed = video_mat - low_rank_video_mat

people_frames = people_bg_removed.reshape((dims[0], dims[1], -1))

fps = video.fps
fig, ax = plt.subplots()
def make_frame(t):
    ax.clear()
    ax.imshow(people_frames[...,int(t*fps)], cmap='gray')
    return mplfig_to_npimage(fig)

animation = mpy.VideoClip(make_frame, duration=int(video.duration-1))
animation.write_videofile('video/people.mp4', fps=fps)

people_out = mpy.VideoFileClip("video/people.mp4")
people_out.subclip(0,50).ipython_display(width=300)

Using sklearn's decomposition SVD

U, S, Vt = decomposition.randomized_svd(video_mat, 10)
print (U.shape, S.shape, Vt.shape)

low_rank = U @ np.diag(S) @ Vt
low_rank.shape

plt.figure(figsize=(10, 10))
plt.imshow(low_rank, aspect=0.02, cmap='gray')

SVD_bg_removed = video_mat-low_rank

plot_two_images(video_mat[:,20].reshape(dims),
                SVD_bg_removed[:, 20].reshape(dims), cmap='gray')
                
 SVD_clipped = np.clip(SVD_bg_removed, None, -10)

plot_two_images(video_mat[:,20].reshape(dims),
                SVD_clipped[:, 20].reshape(dims), cmap='gray')

def SVD_background_removal(clip, frames=5, scale=(320, 240), n_components=2):
    #convert a video clip into a video tensor with SVD background removal 
    
    #format the output dimensions 
    shp = np.array(video.get_frame(0).shape)
    shp[2] = -1
    
    #perform SVD
    A = video_to_matrix(clip, frames=frames, scale=scale)
    U, s, Vh = decomposition.randomized_svd(A, n_components)
    low_rank = U @ np.diag(s) @ Vh
    
    #remove background from input video tensor 
    bg_removed = A-low_rank
    bg_removed = bg_removed.reshape(shp)
    return bg_removed

bg = SVD_background_removal(video, frames=794)
bg.shape

fps = video.fps
fig, ax = plt.subplots()
def make_frame(t):
    ax.clear()
    
    #processed frame 
    img = bg[:, : ,int(t*fps)]
    
    #clip edges (noise)
    img = np.clip(img, None, -10)
    
    #output 
    ax.imshow(img, cmap='gray')
    out = mplfig_to_npimage(fig)[40:250, 78:360, :]
    # out = scipy.misc.imresize(out, (240, 320))    
    out = np.array(Image.fromarray(out).resize((320, 240)))         
        
    return out 

#write to video 
animation = mpy.VideoClip(make_frame, duration=video.duration)
animation.write_videofile('video/Video_003_background_removed.mp4', fps=fps)