annotate metrics_computation.m @ 1:3ea8ed09af0f tip

additional clarifications

% Metrics computations 
% This script assumes the existence of folders "results_<annotator>" , 
% that the system outputs are text files saved in folders 
% "Outputs_<annotator>" and their names are of the form:
% 'Outputs_(bdm/sid)/XX/scriptXX<...>.txt 
% where <...> is: _Bases_X-XX_thres_XXXX_filt_X

clear all

% PARAMETERS
ANOT_FLAG = 1; % possible values: 1 or 2

% INITIALIZATIONS

%Annotations
Annotators = {'_bdm', '_sid'};

% Path to GT annotations
anotpath = ['Development_Set/annotation' num2str(ANOT_FLAG) '/Processed/'];

% Collect all output
AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/01/*txt']);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% Results
results = cell(length(AllOutputs),3);

for i = 1 : length(AllOutputs)
    outputFile = ['Outputs' Annotators{ANOT_FLAG} '/01/' AllOutputs(i).name];
    GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
    % Compute a metric (choose from the 3 below
    [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
    [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
    [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
    resultname = ['results_' AllOutputs(i).name(1:end-4)];
    results{i,1} = resultsEB;
    results{i,1}.eval = 'EB'; 
    results{i,2} = resultsCWEB;
    results{i,2}.eval = 'CWEB';    
    results{i,3} = resultsFB;
    results{i,3}.eval = 'FB';   
    results{i,1}.name = resultname;
    results{i,2}.name = resultname;
    results{i,3}.name = resultname;   
end
save(['results' Annotators{ANOT_FLAG} '/results01.mat'], 'results');
 clear results
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Collect all output
AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/02/*txt']);

% Results
results = cell(length(AllOutputs),3);

for i = 1 : length(AllOutputs)
    outputFile = ['Outputs' Annotators{ANOT_FLAG} '/02/' AllOutputs(i).name];
    GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
    % Compute a metric (choose from the 3 below
    [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
    [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
    [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
    resultname = ['results_' AllOutputs(i).name(1:end-4)];
    results{i,1} = resultsEB;
    results{i,1}.eval = 'EB'; 
    results{i,2} = resultsCWEB;
    results{i,2}.eval = 'CWEB';    
    results{i,3} = resultsFB;
    results{i,3}.eval = 'FB';   
    results{i,1}.name = resultname;
    results{i,2}.name = resultname;
    results{i,3}.name = resultname;   
end
 save(['results' Annotators{ANOT_FLAG} '/results02.mat'], 'results');
 clear results
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Collect all output
AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/03/*txt']);

% Results
results = cell(length(AllOutputs),3);

for i = 1 : length(AllOutputs)
    outputFile = ['Outputs' Annotators{ANOT_FLAG} '/03/' AllOutputs(i).name];
    GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
    % Compute a metric (choose from the 3 below
    [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
    [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
    [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
    resultname = ['results_' AllOutputs(i).name(1:end-4)];
    results{i,1} = resultsEB;
    results{i,1}.eval = 'EB'; 
    results{i,2} = resultsCWEB;
    results{i,2}.eval = 'CWEB';    
    results{i,3} = resultsFB;
    results{i,3}.eval = 'FB';   
    results{i,1}.name = resultname;
    results{i,2}.name = resultname;
    results{i,3}.name = resultname;   
end
save(['results' Annotators{ANOT_FLAG} '/results03.mat'], 'results');
 clear results
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Load all results
res = cell(1);
load(['results' Annotators{ANOT_FLAG} '/results01.mat'])
r1 = results;
load(['results' Annotators{ANOT_FLAG} '/results02.mat']) 
r2 = results;
load(['results' Annotators{ANOT_FLAG} '/results03.mat']) 
r3 = results;

% Calculate mean F-measure values for all development set audio files:
Fmeasure_EB = []; Fmeasure_CWEB = []; Fmeasure_FB = [];
for i=1:size(results,1)
Fmeasure_EB(i)=(r1{i,1}.F+r2{i,1}.F+r3{i,1}.F)/3;
Fmeasure_CWEB(i)=(r1{i,2}.F+r2{i,2}.F+r3{i,2}.F)/3;
Fmeasure_FB(i)=(r1{i,3}.F+r2{i,3}.F+r3{i,3}.F)/3;
end

% Choose the combination of system parameters that maximizes each metric
[valEB, posEB] = max(Fmeasure_EB);
[valCWEB, posCWEB] = max(Fmeasure_CWEB);
[valFB, posFB] = max(Fmeasure_FB);

% Collect the names of the system versions that produced the maximum F-measure 
% for each metric
res = {results{posEB,1}.name; valEB; results{posCWEB,2}.name; valCWEB ...
    ;results{posFB,3}.name; valFB};

% THE FOLLOWING COMPUTES ALL THE METRICS FOR EACH EVAL CASE AND THE OPTIMAL
% PARAMETERS
% Event Based 'EB'
all_res_EB = {(r1{posEB,1}.Rec+r2{posEB,1}.Rec+r3{posEB,1}.Rec)/3;
              (r1{posEB,1}.Pre+r2{posEB,1}.Pre+r3{posEB,1}.Pre)/3;
              (r1{posEB,1}.F+r2{posEB,1}.F+r3{posEB,1}.F)/3;
              (r1{posEB,1}.AEER+r2{posEB,1}.AEER+r3{posEB,1}.AEER)/3;
              (r1{posEB,1}.RecOff+r2{posEB,1}.RecOff+r3{posEB,1}.RecOff)/3;
              (r1{posEB,1}.PreOff+r2{posEB,1}.PreOff+r3{posEB,1}.PreOff)/3;
              (r1{posEB,1}.FOff+r2{posEB,1}.FOff+r3{posEB,1}.FOff)/3;
              (r1{posEB,1}.AEEROff+r2{posEB,1}.AEEROff+r3{posEB,1}.AEEROff)/3};
% Class-wise Event Based 'CWEB'
all_res_CWEB = {(r1{posCWEB,2}.Rec+r2{posCWEB,2}.Rec+r3{posCWEB,2}.Rec)/3;
              (r1{posCWEB,2}.Pre+r2{posCWEB,2}.Pre+r3{posCWEB,2}.Pre)/3;
              (r1{posCWEB,2}.F+r2{posCWEB,2}.F+r3{posCWEB,2}.F)/3;
              (r1{posCWEB,2}.AEER+r2{posCWEB,2}.AEER+r3{posCWEB,2}.AEER)/3;
              (r1{posCWEB,2}.RecOff+r2{posCWEB,2}.RecOff+r3{posCWEB,2}.RecOff)/3;
              (r1{posCWEB,2}.PreOff+r2{posCWEB,2}.PreOff+r3{posCWEB,2}.PreOff)/3;
              (r1{posCWEB,2}.FOff+r2{posCWEB,2}.FOff+r3{posCWEB,2}.FOff)/3;
              (r1{posCWEB,2}.AEEROff+r2{posCWEB,2}.AEEROff+r3{posCWEB,2}.AEEROff)/3};
% Frame Based 'FB'
all_res_FB = {(r1{posFB,3}.Rec+r2{posFB,3}.Rec+r3{posFB,3}.Rec)/3;
              (r1{posFB,3}.Pre+r2{posFB,3}.Pre+r3{posFB,3}.Pre)/3;
              (r1{posFB,3}.F+r2{posFB,3}.F+r3{posFB,3}.F)/3;
              (r1{posFB,3}.AEER+r2{posFB,3}.AEER+r3{posFB,3}.AEER)/3;
              '-';
              '-';
              '-';
              '-';};
          
COMP_METRICS = [{'Rec';'Pre';'F';'AEER';'RecOff';'PreOff';'FOff';'AEEROff'},all_res_EB, all_res_CWEB, all_res_FB]; 

disp('Computed metrics for the Event Based (EB) [column 2],')
disp('the Class-wise Event Based (CWEB) [column 3],')
disp('and the Frame Based (FB) [column 4]')
disp(COMP_METRICS)

% system version names
% disp(res)