%% Action recognition on video via Bag-of-Visual-Words
% Table of Contents
% Getting spatio-temporal (volume) descriptors
%     HOG features (extract)
%     HOF features (extract)
%     Load features (already computed)
%     Building of BoVW out of training videos
%     Build visual vocabulary using k-means
%      K-means Descriptor quantization
%     Represent videos by histograms of quantized features
% Video classification
%     Classify videos with nearest neighbor (NN) classifier
%     Classify videos with SVM classifier (MATLAB default)
% Compare the performances achieved with HoG (Histogram of Gradients) with those given by HOF (Histogram of Optical Flow) on video classification problem
%
%% Getting spatio-temporal (volume) descriptors
%

 

clear
close all

% DATASET
dataset = 'video_actions/';

% PATHS
home_dir = '/Users/grossi/';
dataset_dir = fullfile(home_dir, 'data/', dataset)
source_dir = fullfile(home_dir,'didattica/corsi/IM/MATLAB/video/BoVW/action_lab/')
features_dir = fullfile(source_dir, 'feats/')
addpath(genpath(source_dir)) % add source sub-dirs

% settings
blockSize = [10 10 8]       % block size is 8 by 8 pixels by 6 frames but the number of frames may vary
numBlocks = [2 2 2]       % 3 x 3 spatial blocks and 2 temporal blocks
numOr = 8;                % Quantization in 8 orientations
fsr = 6;                  % fix a sample rate for video frames (FrameSampleRate)
blockSize(3) = 6/fsr;     % Get correct number of frames per block
feat_decim_rate = 0.01;   % visual feature decimation rate (to spare comp. time)

% choose the descriptor
desc = 0
if desc 
    HOF_flag = true;  
    HOG_flag = false;
else
    HOG_flag = true;
    HOF_flag = false;
end

% number of videos selected for training (e.g. 10 for Caltech-101)
num_train_vid = 10    
% number of videos selected for test (e.g. 10 for Caltech-101)
num_test_vid = 10      
% number of codewords (i.e. K for the k-means algorithm)
nwords_codebook = 250

file_ext='mp4';          % video file extension
save_file_flag = true;   % enable data file storing of partial results
load_file_flag = false;  % enable data file loading of partial results

rng(1); % For reproducibility

% DATASET
disp('Loading dataset...');

% Create a new dataset split
data = create_train_test(dataset_dir,num_train_vid,num_test_vid,file_ext);
classes = {data.classname}; % create cell array of class name strings
num_classes = length(classes);
fprintf('  Num classes: %d\n',num_classes);
fprintf('  Num training videos: %d\n',num_train_vid);
fprintf('  Num test videos: %d\n',num_test_vid);

%
%% HOG features (extract)
%
% Get Densely Sampled Histogram of Oriented Gradients volume descriptors from a video
%
% HINT:
% use the function [hogs, info] = Video2DenseHOGVolumes(video, blockSize, numBlocks, numOr)
% Final size of the volume per descriptor is  (blockSize * numBlocks) pixels.
% Oriented Gradients are calculated using HAAR features
% video:                N x M x F grayscale video
% blockSize:         1 x 3 vector with sub-block size in pixels ([pRow pCol pFrames]
% numBlocks:       1 x 3 vector with number of blocks [nRow nCol nZ]
% numOr (optional): Number of orientations for the histogram (default: 8)
% hogs:             Hog descriptors
% info:             Info structure containing e.g. coordinates

 

% Extracting training HOG features
disp('Extracting training HOG features...');
hogTrain = cell(num_classes,num_train_vid);
hogTest = cell(num_classes,num_test_vid);

% For-loop over the training videos for HOG
for c = 1:num_classes
    fprintf(' processing class: %s\n',data(c).classname);
    idx1 = 1; idx2 = 1;
    for v = 1:data(c).n_videos
        file_name = strcat(dataset_dir,data(c).classname, '/',data(c).files{v});
        vid = VideoReadNative(file_name);
        sampledVid = vid(:,:,1:fsr:end); % Subsample framerate of video
        vid = Video2DenseHOGVolumes(sampledVid,blockSize, numBlocks, numOr);
        if data(c).train_id(v)
            % Get HOG descriptors
            hogTrain{c,idx1} = vid;
            idx1 = idx1 + 1;
        else
            hogTest{c,idx2} = vid;
            idx2 = idx2 + 1;
        end
    end
end
fprintf('Done!... Num classes processed: %d \n',c)
if save_file_flag
    save([features_dir 'hogDesc.mat'],'hogTrain','hogTest')
end

%
%% HOF features (extract)
%
% Get Densely Sampled Histogram of Oriented Optical Flow volume descriptors from a video. Final size of the volume per descriptor is (blockSize x numBlocks) pixels.
%  Optical Flow is calculated using MATLABs CV toolbox
%  Soft cell-borders within a single frame (linear interpolation in row and col directions)
%  Hard cell-borders in time direction
%  Sampling is as dense as subcells
% HINT:
% use the function vid = Video2DenseHOFVolumes(sampledVid,blockSize, numBlocks, numOr, flowMethod)
%

 

% Extracting training HOF features
disp('Extracting training HOF features...');
hofTrain = cell(num_classes,num_train_vid);
hofTest = cell(num_classes,num_test_vid);
flowMethod = 'Horn-Schunck'; % the optical flow choice

% For-loop over the training videos for HOG
for c = 1:num_classes
    fprintf(' processing class: %s\n',data(c).classname);
    idx1 = 1; idx2 = 1;
    for v = 1:data(c).n_videos
        file_name = strcat(dataset_dir,data(c).classname, '/',data(c).files{v});
        vid = VideoReadNative(file_name);
        sampledVid = vid(:,:,1:fsr:end); % Subsample framerate of video
        vid = Video2DenseHOFVolumes(sampledVid,blockSize, numBlocks, numOr, flowMethod);
        if data(c).train_id(v)
            % Get HOG descriptors
            hofTrain{c,idx1} = vid;
            idx1 = idx1 + 1;
        else
            hofTest{c,idx2} = vid;
            idx2 = idx2 + 1;
        end
    end
end
fprintf('Done!... Num classes processed: %d \n',c)
if save_file_flag
    save([features_dir 'hofDesc.mat'],'hofTrain','hofTest')
end

%
%% Load features (already computed)
%

%% Load pre-computed HOG features for all videos (to avoid previous step)

load([features_dir 'hofDesc.mat'])
load([features_dir 'hogDesc.mat'])

%
%% Building of BoVW out of training videos
%

 

% concatenate a subsample of descriptors from training videos


%% TODO...


if HOG_flag
    fprintf('BoVW dims: num feats %d - HOG size %d\n',size(BoVW,1),size(BoVW,2))
end
if HOF_flag
    fprintf('BoVW dims: num feats %d - HOF size %d\n',size(BoVW,1),size(BoVW,2))
end

%
%% Build visual vocabulary using k-means
%

 

% sample random descriptors from all training descriptors


%% TODO...

%
%%  K-means Descriptor quantization
%
% K-means descriptor quantization means assignment of each feature descriptor with the identity of its nearest cluster mean, i.e. visual word. Your task is to quantize HOG descriptors in all training and test videos using the visual codebook CODEBOOK constructed above.
%
% TODO:
% compute Euclidean distances between VC and all descriptors in each training and test video
% Hint: to compute all-to-all distance matrix for two sets of descriptors D1 & D2 use dmat = eucliddist(D1,D2);
% compute visual word ID for each feature by minimizing the distance between feature HOG descriptors and VC
% Hint: apply 'min' function to 'dmat' computed above along the dimension (1 or 2) corresponding to VC, e.g.: [mv,visword] = min(dmat,[],2); if you compute dmat as dmat=eucliddist(HOG_mat,VC);
% Structure array 'desc' to represent visual feats for each (train & test) video fields:
% desc(i).visword: N x 144 array with N HOG descriptors
% desc(i).quantdist: file name of original video
% desc(i).label: class label of original video

 

fprintf('\nFeature quantization...');

%% TODO...

fprintf(' done!\n');

%
%%
%% Represent videos by histograms of quantized features
%
% Represent each video by the normalized histogram of visual word labels of its features. Compute word histogram H over the whole video, normalize histograms w.r.t. L1-norm.
%
% TODO:
% For each training and test video compute H.
% Hint: use Matlab function histcounts to compute histograms.

 

disp('Bag-of-Features histograms...');


%% TODO...

disp(' done!')

%
%% Video classification
%
%
%% Classify videos with nearest neighbor (NN) classifier
%

 

method_name = 'NN L2';


% TODO....


% Compute classification accuracy
compute_accuracy(data,labelTest,labelTestNN,classes,method_name,descTest,1,0);

%
%%
%% Classify videos with SVM classifier (MATLAB default)
%
% MATLAB class: Error-Correcting Output Codes Model
% An error-correcting output codes (ECOC) model reduces the problem of classification with three or more classes to a set of binary classifiers.
% ECOC classification requires a coding design, which determines the classes that the binary learners train on, and a decoding scheme, which determines how the results (predictions) of the binary classifiers are aggregated.
%

 

method_name = 'Linear SVM';

% Train an error-correcting output codes (ECOC) multiclass model using
% support vector machine (SVM) binary learners
SVM = fitcecoc(bofTrain,labelTrain);

% SVM is a ClassificationECOC model.
% By default, fitcecoc uses SVM binary learners, and uses a one-versus-one coding design
% You can access Mdl properties using dot notation
SVM.ClassNames  % Display the coding design matrix

% label = predict(Mdl,X) returns a vector of predicted class labels for the predictor data in the table or
% matrix X, based on the full or compact, trained, multiclass, error-correcting output code (ECOC) model SVM
labelTestSVM = predict(SVM,bofTest); % predicted class labels

acc=sum(labelTestSVM == labelTest)/length(labelTest);
fprintf('\n*** %s ***\n  Accuracy = %1.4f%% (classification)\n',method_name,acc*100)

% Compute classification accuracy
compute_accuracy(data,labelTest,labelTestSVM,classes,method_name,descTest,1,0);

%
% Train an ECOC classifier using the SVM binary learners and the default coding design (one-versus-one) Standardize the predictors and save the support vectors
%

t = templateSVM('Standardize',1);  % t is a template object that contains options for SVM classification

% Holdout statistically assesses the accuracies of two classification models.
% The function first compares their predicted labels against the true labels, and then it detects whether
% the difference between the misclassification rates is statistically significant.
CVMdl = fitcecoc(bofTrain,labelTrain,'Holdout',0.20,'Learners',t);
CMdl = CVMdl.Trained{1};                % Extract trained, compact classifier
labelTestSVM1 = predict(CMdl,bofTest);  % predicted class labels

method_name = 'Linear SVM (holdout)';
acc=sum(labelTestSVM1 == labelTest)/length(labelTest);
fprintf('\n*** %s ***\n  Accuracy = %1.4f%% (classification)\n',method_name,acc*100)

% Compute classification accuracy
compute_accuracy(data,labelTest,labelTestSVM1,classes,method_name,descTest,1,0);

%%