wolffd@0
|
1 function [out, stats, features, individual] = test_generic_display_results(file, get_features, show)
|
wolffd@0
|
2 % [out, stats] = test_generic_display_results([file], get_features)
|
wolffd@0
|
3 %
|
wolffd@0
|
4 % [out, stats, features, individual] = test_generic_display_results([file], get_features)
|
wolffd@0
|
5 %
|
wolffd@0
|
6 % displays the finalresults mat file and enables
|
wolffd@0
|
7 % further analysis and duiagnostics of the individual runs
|
wolffd@0
|
8
|
wolffd@0
|
9 features = [];
|
wolffd@0
|
10
|
wolffd@0
|
11 if nargin < 3
|
wolffd@0
|
12 show = 1;
|
wolffd@0
|
13 end
|
wolffd@0
|
14 if nargin < 2
|
wolffd@0
|
15 get_features = 1;
|
wolffd@0
|
16 end
|
wolffd@0
|
17
|
wolffd@0
|
18 global comparison;
|
wolffd@0
|
19 global comparison_ids;
|
wolffd@0
|
20
|
wolffd@0
|
21 if nargin < 1 || isempty(file) || isnumeric(file)
|
wolffd@0
|
22 u = dir();
|
wolffd@0
|
23 u = {u.name};
|
wolffd@0
|
24 [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1);
|
wolffd@0
|
25
|
wolffd@0
|
26 if numel(idx) < 1
|
wolffd@0
|
27 error 'This directory contains no valid test data';
|
wolffd@0
|
28 end
|
wolffd@0
|
29
|
wolffd@0
|
30 if exist('file','var') && isnumeric(file)
|
wolffd@0
|
31 file = u{idx(file)};
|
wolffd@0
|
32 else
|
wolffd@0
|
33 if numel(idx) > 1
|
wolffd@0
|
34 file = u{idx(ask_dataset())};
|
wolffd@0
|
35 else
|
wolffd@0
|
36 file = u{idx(1)};
|
wolffd@0
|
37 end
|
wolffd@0
|
38 end
|
wolffd@0
|
39 end
|
wolffd@0
|
40
|
wolffd@0
|
41
|
wolffd@0
|
42 % ---
|
wolffd@0
|
43 % LOAD THE RESULT DATA
|
wolffd@0
|
44 % We have:
|
wolffd@0
|
45 % Y
|
wolffd@0
|
46 % out.fparams
|
wolffd@0
|
47 % trainparams
|
wolffd@0
|
48 % dataPartition
|
wolffd@0
|
49 % mean_ok_test
|
wolffd@0
|
50 % var_ok_test
|
wolffd@0
|
51 % mean_ok_train
|
wolffd@0
|
52 % ---
|
wolffd@0
|
53 load(file);
|
wolffd@0
|
54
|
wolffd@0
|
55 % compability
|
wolffd@0
|
56 if isfield(out, 'mlrparams')
|
wolffd@0
|
57 for i = 1:numel(out)
|
wolffd@0
|
58 out(i).trainparams = out(i).mlrparams;
|
wolffd@0
|
59 end
|
wolffd@0
|
60 end
|
wolffd@0
|
61
|
wolffd@0
|
62
|
wolffd@0
|
63 % ---
|
wolffd@0
|
64 % % get statistics for feature parameters
|
wolffd@0
|
65 % Visualise the accuracy and variance
|
wolffd@0
|
66 % ---
|
wolffd@0
|
67 if isfield(out, 'inctrain') && show
|
wolffd@0
|
68 for i = 1:numel(out)
|
wolffd@0
|
69
|
wolffd@0
|
70 figure;
|
wolffd@0
|
71 boxplot([out(i).inctrain.mean_ok_test], sqrt([out(i).inctrain.var_ok_test]), [out(i).inctrain.mean_ok_train]);
|
wolffd@0
|
72 set(gca,'XTick',1:numel(out(i).inctrain.trainfrac), ...
|
wolffd@0
|
73 'XTickLabel', out(i).inctrain.trainfrac* 100);
|
wolffd@0
|
74
|
wolffd@0
|
75 xlabel ('fraction of training data');
|
wolffd@0
|
76 title (sprintf('increasing training size test, config %d',i));
|
wolffd@0
|
77 legend('train', 'train weighted', 'test', 'test weighted');
|
wolffd@0
|
78
|
wolffd@0
|
79 end
|
wolffd@0
|
80 end
|
wolffd@0
|
81
|
wolffd@0
|
82
|
wolffd@0
|
83 if numel([out.mean_ok_test]) > 1 && show
|
wolffd@0
|
84
|
wolffd@0
|
85 % plot means % plot std = sqrt(var) % plot training results
|
wolffd@0
|
86 figure;
|
wolffd@0
|
87 boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]);
|
wolffd@0
|
88 title (sprintf('Performance for all configs'));
|
wolffd@0
|
89 end
|
wolffd@0
|
90
|
wolffd@0
|
91
|
wolffd@0
|
92 % ---
|
wolffd@0
|
93 % write max. test success
|
wolffd@0
|
94 % ---
|
wolffd@0
|
95 mean_ok_test = [out.mean_ok_test];
|
wolffd@0
|
96 [val, idx] = max(mean_ok_test(1,:));
|
wolffd@0
|
97 if show
|
wolffd@0
|
98 fprintf(' --- Maximal test set success: nr. %d, %3.2f percent. --- \n', idx, val * 100)
|
wolffd@0
|
99 end
|
wolffd@0
|
100
|
wolffd@0
|
101 % ---
|
wolffd@0
|
102 % display parameter statistics
|
wolffd@0
|
103 % ---
|
wolffd@0
|
104 stats = test_generic_display_param_influence(out, show);
|
wolffd@0
|
105
|
wolffd@0
|
106
|
wolffd@0
|
107 if nargout < 3
|
wolffd@0
|
108 return;
|
wolffd@0
|
109 end
|
wolffd@0
|
110 % ---
|
wolffd@0
|
111 % display statistics and get features
|
wolffd@0
|
112 % for run with best test success
|
wolffd@0
|
113 % ---
|
wolffd@0
|
114 [resfile, featfile] = get_res_filename(out, idx);
|
wolffd@0
|
115
|
wolffd@0
|
116 % ---
|
wolffd@0
|
117 % import features:
|
wolffd@0
|
118 % 1. reset databse
|
wolffd@0
|
119 % 2. import features
|
wolffd@0
|
120 % 3. assign to clip ids as in ranking
|
wolffd@0
|
121 % ---
|
wolffd@0
|
122 if get_features
|
wolffd@0
|
123 type = MTTAudioFeatureDBgen.import_type(featfile);
|
wolffd@0
|
124 db_name = MTTAudioFeatureDBgen.db_name(type);
|
wolffd@0
|
125 eval(sprintf('global %s', db_name));
|
wolffd@0
|
126 eval(sprintf('%s.reset();', db_name));
|
wolffd@0
|
127 eval(sprintf('features = %s.import(featfile);', db_name));
|
wolffd@0
|
128
|
wolffd@0
|
129 if isfield(out,'clip_ids')
|
wolffd@0
|
130 clips = MTTClip(out(1).clip_ids);
|
wolffd@0
|
131 features = clips.features(type);
|
wolffd@0
|
132 end
|
wolffd@0
|
133 end
|
wolffd@0
|
134
|
wolffd@0
|
135 % ---
|
wolffd@0
|
136 % Display Metric Stats
|
wolffd@0
|
137 % tmp = test_mlr_display_metric_stats(individual.out, individual.diag, features);
|
wolffd@0
|
138 % ---
|
wolffd@0
|
139
|
wolffd@0
|
140 if nargout < 4
|
wolffd@0
|
141 return;
|
wolffd@0
|
142 end
|
wolffd@0
|
143 individual = load(resfile);
|
wolffd@0
|
144 for i = 1:numel(out)
|
wolffd@0
|
145
|
wolffd@0
|
146 [resfile, featfile] = get_res_filename(out, i);
|
wolffd@0
|
147
|
wolffd@0
|
148 if get_features
|
wolffd@0
|
149 % reset db and load testing features
|
wolffd@0
|
150 eval(sprintf('global %s', db_name));
|
wolffd@0
|
151 eval(sprintf('%s.reset();', db_name));
|
wolffd@0
|
152 eval(sprintf('%s.import(featfile);', db_name));
|
wolffd@0
|
153 end
|
wolffd@0
|
154
|
wolffd@0
|
155 % load individual results
|
wolffd@0
|
156 if i == 1;
|
wolffd@0
|
157
|
wolffd@0
|
158 individual = load(resfile);
|
wolffd@0
|
159 else
|
wolffd@0
|
160
|
wolffd@0
|
161 individual(i) = load(resfile);
|
wolffd@0
|
162 end
|
wolffd@0
|
163 end
|
wolffd@0
|
164 end
|
wolffd@0
|
165
|
wolffd@0
|
166 function out = ask_dataset()
|
wolffd@0
|
167 % ---
|
wolffd@0
|
168 % displays the parameters of the datasets,
|
wolffd@0
|
169 % and asks for the right one to display
|
wolffd@0
|
170 % ---
|
wolffd@0
|
171 clc;
|
wolffd@0
|
172 u = dir();
|
wolffd@0
|
173 u = {u.name};
|
wolffd@0
|
174 [idx, strpos] = substrcellfind(u, '_params.mat', 1);
|
wolffd@0
|
175
|
wolffd@0
|
176 for i = 1:numel(idx)
|
wolffd@0
|
177 file = u{idx(i)};
|
wolffd@0
|
178 fprintf('------------ Dataset nr. %d --------------\n',i);
|
wolffd@0
|
179 fprintf('Filename: %s\n',file);
|
wolffd@0
|
180 type(file);
|
wolffd@0
|
181 end
|
wolffd@0
|
182
|
wolffd@0
|
183 out = (input('Please choose the dataset number: '));
|
wolffd@0
|
184 end
|
wolffd@0
|
185
|
wolffd@0
|
186
|
wolffd@0
|
187 function [resfile, featfile] = get_res_filename(out, i)
|
wolffd@0
|
188 % get filename given test results and index
|
wolffd@0
|
189
|
wolffd@0
|
190 paramhash = hash(xml_format(out(i).fparams),'MD5');
|
wolffd@0
|
191
|
wolffd@0
|
192 paramhash_mlr = hash(xml_format(out(i).trainparams),'MD5');
|
wolffd@0
|
193
|
wolffd@0
|
194 featfile = sprintf('runlog_%s_feat.mat', paramhash);
|
wolffd@0
|
195
|
wolffd@0
|
196 resfile = sprintf('runlog_%s.%s_results.mat',...
|
wolffd@0
|
197 paramhash, paramhash_mlr);
|
wolffd@0
|
198 end
|
wolffd@0
|
199
|
wolffd@0
|
200
|
wolffd@0
|
201 function boxplot(mean, std, train);
|
wolffd@0
|
202
|
wolffd@0
|
203 bar([train; mean]', 1.5);
|
wolffd@0
|
204 hold on;
|
wolffd@0
|
205 errorbar(1:size(mean,2), mean(1,:), std(1,:),'.');
|
wolffd@0
|
206 % plot(train,'rO');
|
wolffd@0
|
207 colormap(spring);
|
wolffd@0
|
208 axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]);
|
wolffd@0
|
209 end
|