Mercurial > hg > d-case-event

comparison metrics_computation.m @ 0:22b10c5b72e8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial import of complete code and test files
author Dimitrios Giannoulis
date Wed, 13 Mar 2013 10:33:43 +0000
parents
children 3ea8ed09af0f
comparison
equal deleted inserted replaced
-1:000000000000 0:22b10c5b72e8
1 % Metrics computations
2 % This script assumes the existence of folders "results_<annotator>" ,
3 % that the system outputs are text files saved in folders
4 % "Outputs_<annotator>" and their names are of the form:
5 % 'Outputs_(bdm/sid)/XX/scriptXX<...>.txt
6 % where <...> is: _Bases_X-XX_thres_XXXX_filt_X
7
8 clear all
9
10 % PARAMETERS
11 ANOT_FLAG = 1; % possible values: 1 or 2
12
13 % INITIALIZATIONS
14
15 %Annotations
16 Annotators = {'_bdm', '_sid'};
17
18 % Path to GT annotations
19 anotpath = ['Development_Set/annotation' num2str(ANOT_FLAG) '/Processed/'];
20
21 % Collect all output
22 AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/01/*txt']);
23
24 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
25
26 % Results
27 results = cell(length(AllOutputs),3);
28
29 for i = 1 : length(AllOutputs)
30 outputFile = ['Outputs' Annotators{ANOT_FLAG} '/01/' AllOutputs(i).name];
31 GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
32 % Compute a metric (choose from the 3 below
33 [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
34 [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
35 [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
36 resultname = ['results_' AllOutputs(i).name(1:end-4)];
37 results{i,1} = resultsEB;
38 results{i,1}.eval = 'EB';
39 results{i,2} = resultsCWEB;
40 results{i,2}.eval = 'CWEB';
41 results{i,3} = resultsFB;
42 results{i,3}.eval = 'FB';
43 results{i,1}.name = resultname;
44 results{i,2}.name = resultname;
45 results{i,3}.name = resultname;
46 end
47 save(['results' Annotators{ANOT_FLAG} '/results01.mat'], 'results');
48 clear results
49 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
50 % Collect all output
51 AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/02/*txt']);
52
53 % Results
54 results = cell(length(AllOutputs),3);
55
56 for i = 1 : length(AllOutputs)
57 outputFile = ['Outputs' Annotators{ANOT_FLAG} '/02/' AllOutputs(i).name];
58 GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
59 % Compute a metric (choose from the 3 below
60 [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
61 [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
62 [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
63 resultname = ['results_' AllOutputs(i).name(1:end-4)];
64 results{i,1} = resultsEB;
65 results{i,1}.eval = 'EB';
66 results{i,2} = resultsCWEB;
67 results{i,2}.eval = 'CWEB';
68 results{i,3} = resultsFB;
69 results{i,3}.eval = 'FB';
70 results{i,1}.name = resultname;
71 results{i,2}.name = resultname;
72 results{i,3}.name = resultname;
73 end
74 save(['results' Annotators{ANOT_FLAG} '/results02.mat'], 'results');
75 clear results
76 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
77 % Collect all output
78 AllOutputs = dir(['Outputs' Annotators{ANOT_FLAG} '/03/*txt']);
79
80 % Results
81 results = cell(length(AllOutputs),3);
82
83 for i = 1 : length(AllOutputs)
84 outputFile = ['Outputs' Annotators{ANOT_FLAG} '/03/' AllOutputs(i).name];
85 GTFile = [anotpath AllOutputs(i).name(1:8) Annotators{ANOT_FLAG} '.txt'];
86 % Compute a metric (choose from the 3 below
87 [resultsEB] = eventDetectionMetrics_eventBased(outputFile,GTFile);
88 [resultsCWEB] = eventDetectionMetrics_classWiseEventBased(outputFile,GTFile);
89 [resultsFB] = eventDetectionMetrics_frameBased(outputFile,GTFile);
90 resultname = ['results_' AllOutputs(i).name(1:end-4)];
91 results{i,1} = resultsEB;
92 results{i,1}.eval = 'EB';
93 results{i,2} = resultsCWEB;
94 results{i,2}.eval = 'CWEB';
95 results{i,3} = resultsFB;
96 results{i,3}.eval = 'FB';
97 results{i,1}.name = resultname;
98 results{i,2}.name = resultname;
99 results{i,3}.name = resultname;
100 end
101 save(['results' Annotators{ANOT_FLAG} '/results03.mat'], 'results');
102 clear results
103 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
104 % Load all results
105 res = cell(1);
106 load(['results' Annotators{ANOT_FLAG} '/results01.mat'])
107 r1 = results;
108 load(['results' Annotators{ANOT_FLAG} '/results02.mat'])
109 r2 = results;
110 load(['results' Annotators{ANOT_FLAG} '/results03.mat'])
111 r3 = results;
112
113 Fmeasure_EB = []; Fmeasure_CWEB = []; Fmeasure_FB = [];
114 for i=1:size(results,1)
115 Fmeasure_EB(i)=(r1{i,1}.F+r2{i,1}.F+r3{i,1}.F)/3;
116 Fmeasure_CWEB(i)=(r1{i,2}.F+r2{i,2}.F+r3{i,2}.F)/3;
117 Fmeasure_FB(i)=(r1{i,3}.F+r2{i,3}.F+r3{i,3}.F)/3;
118 end
119
120 [valEB, posEB] = max(Fmeasure_EB);
121 [valCWEB, posCWEB] = max(Fmeasure_CWEB);
122 [valFB, posFB] = max(Fmeasure_FB);
123
124 res = {results{posEB,1}.name; valEB; results{posCWEB,2}.name; valCWEB ...
125 ;results{posFB,3}.name; valFB};
126
127 % THE FOLLOWING COMPUTES ALL THE METRICS FOR EACH EVAL CASE AND THE OPTIMAL
128 % PARAMETERS
129 'EB'
130 all_res_EB = {(r1{posEB,1}.Rec+r2{posEB,1}.Rec+r3{posEB,1}.Rec)/3;
131 (r1{posEB,1}.Pre+r2{posEB,1}.Pre+r3{posEB,1}.Pre)/3;
132 (r1{posEB,1}.F+r2{posEB,1}.F+r3{posEB,1}.F)/3;
133 (r1{posEB,1}.AEER+r2{posEB,1}.AEER+r3{posEB,1}.AEER)/3;
134 (r1{posEB,1}.RecOff+r2{posEB,1}.RecOff+r3{posEB,1}.RecOff)/3;
135 (r1{posEB,1}.PreOff+r2{posEB,1}.PreOff+r3{posEB,1}.PreOff)/3;
136 (r1{posEB,1}.FOff+r2{posEB,1}.FOff+r3{posEB,1}.FOff)/3;
137 (r1{posEB,1}.AEEROff+r2{posEB,1}.AEEROff+r3{posEB,1}.AEEROff)/3};
138 'CWEB'
139 all_res_CWEB = {(r1{posCWEB,2}.Rec+r2{posCWEB,2}.Rec+r3{posCWEB,2}.Rec)/3;
140 (r1{posCWEB,2}.Pre+r2{posCWEB,2}.Pre+r3{posCWEB,2}.Pre)/3;
141 (r1{posCWEB,2}.F+r2{posCWEB,2}.F+r3{posCWEB,2}.F)/3;
142 (r1{posCWEB,2}.AEER+r2{posCWEB,2}.AEER+r3{posCWEB,2}.AEER)/3;
143 (r1{posCWEB,2}.RecOff+r2{posCWEB,2}.RecOff+r3{posCWEB,2}.RecOff)/3;
144 (r1{posCWEB,2}.PreOff+r2{posCWEB,2}.PreOff+r3{posCWEB,2}.PreOff)/3;
145 (r1{posCWEB,2}.FOff+r2{posCWEB,2}.FOff+r3{posCWEB,2}.FOff)/3;
146 (r1{posCWEB,2}.AEEROff+r2{posCWEB,2}.AEEROff+r3{posCWEB,2}.AEEROff)/3};
147 'FB'
148 all_res_FB = {(r1{posFB,3}.Rec+r2{posFB,3}.Rec+r3{posFB,3}.Rec)/3;
149 (r1{posFB,3}.Pre+r2{posFB,3}.Pre+r3{posFB,3}.Pre)/3;
150 (r1{posFB,3}.F+r2{posFB,3}.F+r3{posFB,3}.F)/3;
151 (r1{posFB,3}.AEER+r2{posFB,3}.AEER+r3{posFB,3}.AEER)/3;
152 NaN;
153 NaN;
154 NaN;
155 NaN;};
156 ALL_METRICS = [all_res_EB, all_res_CWEB, all_res_FB]