Dimitrios@0: % Metrics computations 
Dimitrios@0: % This script assumes the existence of folders "results_<annotator>" , 
Dimitrios@0: % that the system outputs are text files saved in folders 
Dimitrios@0: % "Outputs_<annotator>" and their names are of the form:
Dimitrios@0: % 'Outputs_(bdm/sid)/XX/scriptXX<...>.txt 
Dimitrios@0: % where <...> is: _Bases_X-XX_thres_XXXX_filt_X
Dimitrios@0: 
Dimitrios@0: clear all
Dimitrios@0: 
Dimitrios@0: % PARAMETERS
Dimitrios@0: ANOT_FLAG = 1; % possible values: 1 or 2
Dimitrios@0: 
Dimitrios@0: % INITIALIZATIONS
Dimitrios@0: 
Dimitrios@0: %Annotations
Dimitrios@0: Annotators = {'_bdm', '_sid'};
Dimitrios@0: 
Dimitrios@0: % Path to GT annotations
Dimitrios@0: anotpath = ['Development_Set/annotation' num2str(ANOT_FLAG) '/Processed/'];
Dimitrios@0: 
Dimitrios@0: % Collect all output
Dimitrios@0: AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/01/*txt']);
Dimitrios@0: 
Dimitrios@0: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Dimitrios@0: 
Dimitrios@0: % Results
Dimitrios@0: results = cell(length(AllOutputs),3);
Dimitrios@0: 
Dimitrios@0: for i = 1 : length(AllOutputs)
Dimitrios@0:     outputFile = ['Outputs' Annotators{ANOT_FLAG} '/01/' AllOutputs(i).name];
Dimitrios@0:     GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
Dimitrios@0:     % Compute a metric (choose from the 3 below
Dimitrios@0:     [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
Dimitrios@0:     [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
Dimitrios@0:     [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
Dimitrios@0:     resultname = ['results_' AllOutputs(i).name(1:end-4)];
Dimitrios@0:     results{i,1} = resultsEB;
Dimitrios@0:     results{i,1}.eval = 'EB'; 
Dimitrios@0:     results{i,2} = resultsCWEB;
Dimitrios@0:     results{i,2}.eval = 'CWEB';    
Dimitrios@0:     results{i,3} = resultsFB;
Dimitrios@0:     results{i,3}.eval = 'FB';   
Dimitrios@0:     results{i,1}.name = resultname;
Dimitrios@0:     results{i,2}.name = resultname;
Dimitrios@0:     results{i,3}.name = resultname;   
Dimitrios@0: end
Dimitrios@0: save(['results' Annotators{ANOT_FLAG} '/results01.mat'], 'results');
Dimitrios@0:  clear results
Dimitrios@0: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Dimitrios@0: % Collect all output
Dimitrios@0: AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/02/*txt']);
Dimitrios@0: 
Dimitrios@0: % Results
Dimitrios@0: results = cell(length(AllOutputs),3);
Dimitrios@0: 
Dimitrios@0: for i = 1 : length(AllOutputs)
Dimitrios@0:     outputFile = ['Outputs' Annotators{ANOT_FLAG} '/02/' AllOutputs(i).name];
Dimitrios@0:     GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
Dimitrios@0:     % Compute a metric (choose from the 3 below
Dimitrios@0:     [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
Dimitrios@0:     [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
Dimitrios@0:     [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
Dimitrios@0:     resultname = ['results_' AllOutputs(i).name(1:end-4)];
Dimitrios@0:     results{i,1} = resultsEB;
Dimitrios@0:     results{i,1}.eval = 'EB'; 
Dimitrios@0:     results{i,2} = resultsCWEB;
Dimitrios@0:     results{i,2}.eval = 'CWEB';    
Dimitrios@0:     results{i,3} = resultsFB;
Dimitrios@0:     results{i,3}.eval = 'FB';   
Dimitrios@0:     results{i,1}.name = resultname;
Dimitrios@0:     results{i,2}.name = resultname;
Dimitrios@0:     results{i,3}.name = resultname;   
Dimitrios@0: end
Dimitrios@0:  save(['results' Annotators{ANOT_FLAG} '/results02.mat'], 'results');
Dimitrios@0:  clear results
Dimitrios@0: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Dimitrios@0: % Collect all output
Dimitrios@0: AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/03/*txt']);
Dimitrios@0: 
Dimitrios@0: % Results
Dimitrios@0: results = cell(length(AllOutputs),3);
Dimitrios@0: 
Dimitrios@0: for i = 1 : length(AllOutputs)
Dimitrios@0:     outputFile = ['Outputs' Annotators{ANOT_FLAG} '/03/' AllOutputs(i).name];
Dimitrios@0:     GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
Dimitrios@0:     % Compute a metric (choose from the 3 below
Dimitrios@0:     [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
Dimitrios@0:     [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
Dimitrios@0:     [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
Dimitrios@0:     resultname = ['results_' AllOutputs(i).name(1:end-4)];
Dimitrios@0:     results{i,1} = resultsEB;
Dimitrios@0:     results{i,1}.eval = 'EB'; 
Dimitrios@0:     results{i,2} = resultsCWEB;
Dimitrios@0:     results{i,2}.eval = 'CWEB';    
Dimitrios@0:     results{i,3} = resultsFB;
Dimitrios@0:     results{i,3}.eval = 'FB';   
Dimitrios@0:     results{i,1}.name = resultname;
Dimitrios@0:     results{i,2}.name = resultname;
Dimitrios@0:     results{i,3}.name = resultname;   
Dimitrios@0: end
Dimitrios@0: save(['results' Annotators{ANOT_FLAG} '/results03.mat'], 'results');
Dimitrios@0:  clear results
Dimitrios@0: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Dimitrios@0: % Load all results
Dimitrios@0: res = cell(1);
Dimitrios@0: load(['results' Annotators{ANOT_FLAG} '/results01.mat'])
Dimitrios@0: r1 = results;
Dimitrios@0: load(['results' Annotators{ANOT_FLAG} '/results02.mat']) 
Dimitrios@0: r2 = results;
Dimitrios@0: load(['results' Annotators{ANOT_FLAG} '/results03.mat']) 
Dimitrios@0: r3 = results;
Dimitrios@0: 
Dimitrios@1: % Calculate mean F-measure values for all development set audio files:
Dimitrios@0: Fmeasure_EB = []; Fmeasure_CWEB = []; Fmeasure_FB = [];
Dimitrios@0: for i=1:size(results,1)
Dimitrios@0: Fmeasure_EB(i)=(r1{i,1}.F+r2{i,1}.F+r3{i,1}.F)/3;
Dimitrios@0: Fmeasure_CWEB(i)=(r1{i,2}.F+r2{i,2}.F+r3{i,2}.F)/3;
Dimitrios@0: Fmeasure_FB(i)=(r1{i,3}.F+r2{i,3}.F+r3{i,3}.F)/3;
Dimitrios@0: end
Dimitrios@0: 
Dimitrios@1: % Choose the combination of system parameters that maximizes each metric
Dimitrios@0: [valEB, posEB] = max(Fmeasure_EB);
Dimitrios@0: [valCWEB, posCWEB] = max(Fmeasure_CWEB);
Dimitrios@0: [valFB, posFB] = max(Fmeasure_FB);
Dimitrios@0: 
Dimitrios@1: % Collect the names of the system versions that produced the maximum F-measure 
Dimitrios@1: % for each metric
Dimitrios@0: res = {results{posEB,1}.name; valEB; results{posCWEB,2}.name; valCWEB ...
Dimitrios@0:     ;results{posFB,3}.name; valFB};
Dimitrios@0: 
Dimitrios@0: % THE FOLLOWING COMPUTES ALL THE METRICS FOR EACH EVAL CASE AND THE OPTIMAL
Dimitrios@0: % PARAMETERS
Dimitrios@1: % Event Based 'EB'
Dimitrios@0: all_res_EB = {(r1{posEB,1}.Rec+r2{posEB,1}.Rec+r3{posEB,1}.Rec)/3;
Dimitrios@0:               (r1{posEB,1}.Pre+r2{posEB,1}.Pre+r3{posEB,1}.Pre)/3;
Dimitrios@0:               (r1{posEB,1}.F+r2{posEB,1}.F+r3{posEB,1}.F)/3;
Dimitrios@0:               (r1{posEB,1}.AEER+r2{posEB,1}.AEER+r3{posEB,1}.AEER)/3;
Dimitrios@0:               (r1{posEB,1}.RecOff+r2{posEB,1}.RecOff+r3{posEB,1}.RecOff)/3;
Dimitrios@0:               (r1{posEB,1}.PreOff+r2{posEB,1}.PreOff+r3{posEB,1}.PreOff)/3;
Dimitrios@0:               (r1{posEB,1}.FOff+r2{posEB,1}.FOff+r3{posEB,1}.FOff)/3;
Dimitrios@0:               (r1{posEB,1}.AEEROff+r2{posEB,1}.AEEROff+r3{posEB,1}.AEEROff)/3};
Dimitrios@1: % Class-wise Event Based 'CWEB'
Dimitrios@0: all_res_CWEB = {(r1{posCWEB,2}.Rec+r2{posCWEB,2}.Rec+r3{posCWEB,2}.Rec)/3;
Dimitrios@0:               (r1{posCWEB,2}.Pre+r2{posCWEB,2}.Pre+r3{posCWEB,2}.Pre)/3;
Dimitrios@0:               (r1{posCWEB,2}.F+r2{posCWEB,2}.F+r3{posCWEB,2}.F)/3;
Dimitrios@0:               (r1{posCWEB,2}.AEER+r2{posCWEB,2}.AEER+r3{posCWEB,2}.AEER)/3;
Dimitrios@0:               (r1{posCWEB,2}.RecOff+r2{posCWEB,2}.RecOff+r3{posCWEB,2}.RecOff)/3;
Dimitrios@0:               (r1{posCWEB,2}.PreOff+r2{posCWEB,2}.PreOff+r3{posCWEB,2}.PreOff)/3;
Dimitrios@0:               (r1{posCWEB,2}.FOff+r2{posCWEB,2}.FOff+r3{posCWEB,2}.FOff)/3;
Dimitrios@0:               (r1{posCWEB,2}.AEEROff+r2{posCWEB,2}.AEEROff+r3{posCWEB,2}.AEEROff)/3};
Dimitrios@1: % Frame Based 'FB'
Dimitrios@0: all_res_FB = {(r1{posFB,3}.Rec+r2{posFB,3}.Rec+r3{posFB,3}.Rec)/3;
Dimitrios@0:               (r1{posFB,3}.Pre+r2{posFB,3}.Pre+r3{posFB,3}.Pre)/3;
Dimitrios@0:               (r1{posFB,3}.F+r2{posFB,3}.F+r3{posFB,3}.F)/3;
Dimitrios@0:               (r1{posFB,3}.AEER+r2{posFB,3}.AEER+r3{posFB,3}.AEER)/3;
Dimitrios@1:               '-';
Dimitrios@1:               '-';
Dimitrios@1:               '-';
Dimitrios@1:               '-';};
Dimitrios@1:           
Dimitrios@1: COMP_METRICS = [{'Rec';'Pre';'F';'AEER';'RecOff';'PreOff';'FOff';'AEEROff'},all_res_EB, all_res_CWEB, all_res_FB]; 
Dimitrios@1: 
Dimitrios@1: disp('Computed metrics for the Event Based (EB) [column 2],')
Dimitrios@1: disp('the Class-wise Event Based (CWEB) [column 3],')
Dimitrios@1: disp('and the Frame Based (FB) [column 4]')
Dimitrios@1: disp(COMP_METRICS)
Dimitrios@1: 
Dimitrios@1: % system version names
Dimitrios@1: % disp(res)