Mercurial > hg > camir-aes2014
comparison core/magnatagatune/tests_evals/test_generic_display_results.m @ 0:e9a9cd732c1e tip
first hg version after svn
author | wolffd |
---|---|
date | Tue, 10 Feb 2015 15:05:51 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:e9a9cd732c1e |
---|---|
1 function [out, stats, features, individual] = test_generic_display_results(file, get_features, show) | |
2 % [out, stats] = test_generic_display_results([file], get_features) | |
3 % | |
4 % [out, stats, features, individual] = test_generic_display_results([file], get_features) | |
5 % | |
6 % displays the finalresults mat file and enables | |
7 % further analysis and duiagnostics of the individual runs | |
8 | |
9 features = []; | |
10 | |
11 if nargin < 3 | |
12 show = 1; | |
13 end | |
14 if nargin < 2 | |
15 get_features = 1; | |
16 end | |
17 | |
18 global comparison; | |
19 global comparison_ids; | |
20 | |
21 if nargin < 1 || isempty(file) || isnumeric(file) | |
22 u = dir(); | |
23 u = {u.name}; | |
24 [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1); | |
25 | |
26 if numel(idx) < 1 | |
27 error 'This directory contains no valid test data'; | |
28 end | |
29 | |
30 if exist('file','var') && isnumeric(file) | |
31 file = u{idx(file)}; | |
32 else | |
33 if numel(idx) > 1 | |
34 file = u{idx(ask_dataset())}; | |
35 else | |
36 file = u{idx(1)}; | |
37 end | |
38 end | |
39 end | |
40 | |
41 | |
42 % --- | |
43 % LOAD THE RESULT DATA | |
44 % We have: | |
45 % Y | |
46 % out.fparams | |
47 % trainparams | |
48 % dataPartition | |
49 % mean_ok_test | |
50 % var_ok_test | |
51 % mean_ok_train | |
52 % --- | |
53 load(file); | |
54 | |
55 % compability | |
56 if isfield(out, 'mlrparams') | |
57 for i = 1:numel(out) | |
58 out(i).trainparams = out(i).mlrparams; | |
59 end | |
60 end | |
61 | |
62 | |
63 % --- | |
64 % % get statistics for feature parameters | |
65 % Visualise the accuracy and variance | |
66 % --- | |
67 if isfield(out, 'inctrain') && show | |
68 for i = 1:numel(out) | |
69 | |
70 figure; | |
71 boxplot([out(i).inctrain.mean_ok_test], sqrt([out(i).inctrain.var_ok_test]), [out(i).inctrain.mean_ok_train]); | |
72 set(gca,'XTick',1:numel(out(i).inctrain.trainfrac), ... | |
73 'XTickLabel', out(i).inctrain.trainfrac* 100); | |
74 | |
75 xlabel ('fraction of training data'); | |
76 title (sprintf('increasing training size test, config %d',i)); | |
77 legend('train', 'train weighted', 'test', 'test weighted'); | |
78 | |
79 end | |
80 end | |
81 | |
82 | |
83 if numel([out.mean_ok_test]) > 1 && show | |
84 | |
85 % plot means % plot std = sqrt(var) % plot training results | |
86 figure; | |
87 boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]); | |
88 title (sprintf('Performance for all configs')); | |
89 end | |
90 | |
91 | |
92 % --- | |
93 % write max. test success | |
94 % --- | |
95 mean_ok_test = [out.mean_ok_test]; | |
96 [val, idx] = max(mean_ok_test(1,:)); | |
97 if show | |
98 fprintf(' --- Maximal test set success: nr. %d, %3.2f percent. --- \n', idx, val * 100) | |
99 end | |
100 | |
101 % --- | |
102 % display parameter statistics | |
103 % --- | |
104 stats = test_generic_display_param_influence(out, show); | |
105 | |
106 | |
107 if nargout < 3 | |
108 return; | |
109 end | |
110 % --- | |
111 % display statistics and get features | |
112 % for run with best test success | |
113 % --- | |
114 [resfile, featfile] = get_res_filename(out, idx); | |
115 | |
116 % --- | |
117 % import features: | |
118 % 1. reset databse | |
119 % 2. import features | |
120 % 3. assign to clip ids as in ranking | |
121 % --- | |
122 if get_features | |
123 type = MTTAudioFeatureDBgen.import_type(featfile); | |
124 db_name = MTTAudioFeatureDBgen.db_name(type); | |
125 eval(sprintf('global %s', db_name)); | |
126 eval(sprintf('%s.reset();', db_name)); | |
127 eval(sprintf('features = %s.import(featfile);', db_name)); | |
128 | |
129 if isfield(out,'clip_ids') | |
130 clips = MTTClip(out(1).clip_ids); | |
131 features = clips.features(type); | |
132 end | |
133 end | |
134 | |
135 % --- | |
136 % Display Metric Stats | |
137 % tmp = test_mlr_display_metric_stats(individual.out, individual.diag, features); | |
138 % --- | |
139 | |
140 if nargout < 4 | |
141 return; | |
142 end | |
143 individual = load(resfile); | |
144 for i = 1:numel(out) | |
145 | |
146 [resfile, featfile] = get_res_filename(out, i); | |
147 | |
148 if get_features | |
149 % reset db and load testing features | |
150 eval(sprintf('global %s', db_name)); | |
151 eval(sprintf('%s.reset();', db_name)); | |
152 eval(sprintf('%s.import(featfile);', db_name)); | |
153 end | |
154 | |
155 % load individual results | |
156 if i == 1; | |
157 | |
158 individual = load(resfile); | |
159 else | |
160 | |
161 individual(i) = load(resfile); | |
162 end | |
163 end | |
164 end | |
165 | |
166 function out = ask_dataset() | |
167 % --- | |
168 % displays the parameters of the datasets, | |
169 % and asks for the right one to display | |
170 % --- | |
171 clc; | |
172 u = dir(); | |
173 u = {u.name}; | |
174 [idx, strpos] = substrcellfind(u, '_params.mat', 1); | |
175 | |
176 for i = 1:numel(idx) | |
177 file = u{idx(i)}; | |
178 fprintf('------------ Dataset nr. %d --------------\n',i); | |
179 fprintf('Filename: %s\n',file); | |
180 type(file); | |
181 end | |
182 | |
183 out = (input('Please choose the dataset number: ')); | |
184 end | |
185 | |
186 | |
187 function [resfile, featfile] = get_res_filename(out, i) | |
188 % get filename given test results and index | |
189 | |
190 paramhash = hash(xml_format(out(i).fparams),'MD5'); | |
191 | |
192 paramhash_mlr = hash(xml_format(out(i).trainparams),'MD5'); | |
193 | |
194 featfile = sprintf('runlog_%s_feat.mat', paramhash); | |
195 | |
196 resfile = sprintf('runlog_%s.%s_results.mat',... | |
197 paramhash, paramhash_mlr); | |
198 end | |
199 | |
200 | |
201 function boxplot(mean, std, train); | |
202 | |
203 bar([train; mean]', 1.5); | |
204 hold on; | |
205 errorbar(1:size(mean,2), mean(1,:), std(1,:),'.'); | |
206 % plot(train,'rO'); | |
207 colormap(spring); | |
208 axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]); | |
209 end |