wolffd@0
|
1 function stats = test_generic_display_param_influence(results, show)
|
wolffd@0
|
2 % returns the mean accuracy influence of each feature and training parameter
|
wolffd@0
|
3 %
|
wolffd@0
|
4 % the influence is measured by comparing the mean
|
wolffd@0
|
5 % achieved accuracy for all tries with each specific
|
wolffd@0
|
6 % parameter being constant
|
wolffd@0
|
7 %
|
wolffd@0
|
8 % TODO: evaluate how the comparisons of all configuration
|
wolffd@0
|
9 % tuples twith just the specific analysed parameter
|
wolffd@0
|
10 % changing differ from the above approach
|
wolffd@0
|
11
|
wolffd@0
|
12 % get statistics for feature parameters
|
wolffd@0
|
13 stats.fparams = gen_param_influence(results, 'fparams');
|
wolffd@0
|
14
|
wolffd@0
|
15 % get statistics for feature parameters
|
wolffd@0
|
16 if isfield(results, 'trainparams')
|
wolffd@0
|
17
|
wolffd@0
|
18 stats.trainparams = gen_param_influence(results, 'trainparams');
|
wolffd@0
|
19
|
wolffd@0
|
20 % the following case is for backwards compability
|
wolffd@0
|
21 elseif isfield(results, 'mlrparams')
|
wolffd@0
|
22
|
wolffd@0
|
23 stats.trainparams = gen_param_influence(results, 'mlrparams');
|
wolffd@0
|
24 end
|
wolffd@0
|
25
|
wolffd@0
|
26 if show
|
wolffd@0
|
27 % display results
|
wolffd@0
|
28
|
wolffd@0
|
29 if ~isempty(stats.fparams)
|
wolffd@0
|
30 figure;
|
wolffd@0
|
31 % subplot(2,1,1);
|
wolffd@0
|
32 display_param_influence(stats.fparams);
|
wolffd@0
|
33 end
|
wolffd@0
|
34
|
wolffd@0
|
35 if ~isempty(stats.trainparams)
|
wolffd@0
|
36 figure;
|
wolffd@0
|
37 % subplot(2,1,2);
|
wolffd@0
|
38 display_param_influence(stats.trainparams);
|
wolffd@0
|
39 end
|
wolffd@0
|
40 end
|
wolffd@0
|
41
|
wolffd@0
|
42 end
|
wolffd@0
|
43
|
wolffd@0
|
44 % ---
|
wolffd@0
|
45 % gen_param_influence
|
wolffd@0
|
46 % ---
|
wolffd@0
|
47 function stats = gen_param_influence(results, paramname)
|
wolffd@0
|
48 % generates statistics given results and parameter type as string.
|
wolffd@0
|
49
|
wolffd@0
|
50 % get individual fields of this parameter set
|
wolffd@0
|
51 ptypes = fieldnames(results(1).(paramname));
|
wolffd@0
|
52
|
wolffd@0
|
53 for i = 1:numel(ptypes)
|
wolffd@0
|
54 % ---
|
wolffd@0
|
55 % get all individual configurations of this parameter.
|
wolffd@0
|
56 % ---
|
wolffd@0
|
57 allvals = [results.(paramname)];
|
wolffd@0
|
58
|
wolffd@0
|
59 % take care of string args
|
wolffd@0
|
60 if ~ischar(allvals(1).(ptypes{i}))
|
wolffd@0
|
61 if ~iscell(allvals(1).(ptypes{i}))
|
wolffd@0
|
62
|
wolffd@0
|
63 % parameter array of chars
|
wolffd@0
|
64 allvals = [allvals.(ptypes{i})];
|
wolffd@0
|
65 else
|
wolffd@0
|
66 % complex parameter array of cells
|
wolffd@0
|
67 for j=1:numel(allvals)
|
wolffd@0
|
68 tmpvals{j} = cell2str(allvals(j).(ptypes{i}));
|
wolffd@0
|
69 end
|
wolffd@0
|
70 allvals = tmpvals;
|
wolffd@0
|
71 end
|
wolffd@0
|
72 else
|
wolffd@0
|
73 % parameter array of numbers
|
wolffd@0
|
74 allvals = {allvals.(ptypes{i})};
|
wolffd@0
|
75 end
|
wolffd@0
|
76
|
wolffd@0
|
77 % save using original parameter name
|
wolffd@0
|
78 tmp = param_influence(results, allvals);
|
wolffd@0
|
79
|
wolffd@0
|
80 if ~isempty(tmp)
|
wolffd@0
|
81 stats.(ptypes{i}) = tmp;
|
wolffd@0
|
82 end
|
wolffd@0
|
83 end
|
wolffd@0
|
84
|
wolffd@0
|
85 if ~exist('stats','var')
|
wolffd@0
|
86 stats = [];
|
wolffd@0
|
87 end
|
wolffd@0
|
88
|
wolffd@0
|
89 end
|
wolffd@0
|
90
|
wolffd@0
|
91
|
wolffd@0
|
92 % ---
|
wolffd@0
|
93 % param_influence
|
wolffd@0
|
94 % ---
|
wolffd@0
|
95 function out = param_influence(results, allvals)
|
wolffd@0
|
96 % give the influence (given results) for the parameter settings
|
wolffd@0
|
97 % given in allvals.
|
wolffd@0
|
98 %
|
wolffd@0
|
99 % numel(results) = numel(allvals)
|
wolffd@0
|
100
|
wolffd@0
|
101 % ---
|
wolffd@0
|
102 % get all different settings of this parameter.
|
wolffd@0
|
103 % NOTE: this might also work results-of the box for strings.
|
wolffd@0
|
104 % not tested, below has to be changed ot cell / matrix notations
|
wolffd@0
|
105 % ---
|
wolffd@0
|
106 entries = unique(allvals);
|
wolffd@0
|
107
|
wolffd@0
|
108 % just calculate for params with more than one option
|
wolffd@0
|
109 if numel(entries) < 2 || ischar(entries)
|
wolffd@0
|
110
|
wolffd@0
|
111 out = [];
|
wolffd@0
|
112 return;
|
wolffd@0
|
113 end
|
wolffd@0
|
114
|
wolffd@0
|
115 % calculate statstics for this fixed parameter
|
wolffd@0
|
116 for j = 1:numel(entries)
|
wolffd@0
|
117
|
wolffd@0
|
118 % care for string parameters
|
wolffd@0
|
119 if ~(iscell(allvals) && ischar(allvals{1}))
|
wolffd@0
|
120 valid_idx = (allvals == entries(j));
|
wolffd@0
|
121
|
wolffd@0
|
122 % mean_ok_test
|
wolffd@0
|
123 valid_ids = find(valid_idx);
|
wolffd@0
|
124 else
|
wolffd@0
|
125 valid_ids = strcellfind(allvals, entries{j}, 1);
|
wolffd@0
|
126 end
|
wolffd@0
|
127
|
wolffd@0
|
128 % ---
|
wolffd@0
|
129 % get the relevant statistics over the variations
|
wolffd@0
|
130 % of the further parameters
|
wolffd@0
|
131 % ---
|
wolffd@0
|
132 mean_ok_testval = [];
|
wolffd@0
|
133 for i = 1:numel(valid_ids)
|
wolffd@0
|
134 mean_ok_testval = [mean_ok_testval results(valid_ids(i)).mean_ok_test(1,:)];
|
wolffd@0
|
135 end
|
wolffd@0
|
136
|
wolffd@0
|
137 [ma,maidx] = max(mean_ok_testval);
|
wolffd@0
|
138 [mi,miidx] = min(mean_ok_testval);
|
wolffd@0
|
139 [me] = mean(mean_ok_testval);
|
wolffd@0
|
140 mean_ok_test(j) = struct('max',ma , ...
|
wolffd@0
|
141 'max_idx',valid_ids(maidx) , ...
|
wolffd@0
|
142 'min',mi , ...
|
wolffd@0
|
143 'min_idx',valid_ids(miidx) , ...
|
wolffd@0
|
144 'mean',me);
|
wolffd@0
|
145
|
wolffd@0
|
146 % ---
|
wolffd@0
|
147 % get the training statistics over the variations
|
wolffd@0
|
148 % of the further parameters
|
wolffd@0
|
149 % ---
|
wolffd@0
|
150 mean_ok_trainval = [];
|
wolffd@0
|
151 for i = 1:numel(valid_ids)
|
wolffd@0
|
152 mean_ok_trainval = [mean_ok_trainval results(valid_ids(i)).mean_ok_train(1,:)];
|
wolffd@0
|
153 end
|
wolffd@0
|
154
|
wolffd@0
|
155 [ma,maidx] = max(mean_ok_trainval);
|
wolffd@0
|
156 % ---
|
wolffd@0
|
157 % NOTE :this allowed for accesment of improvement by RBM selection
|
wolffd@0
|
158 % warning testing random idx instead of best one
|
wolffd@0
|
159 % maidx = max(1, round(rand(1)* numel(valid_ids)));
|
wolffd@0
|
160 % % ---
|
wolffd@0
|
161
|
wolffd@0
|
162 [mi,miidx] = min(mean_ok_trainval);
|
wolffd@0
|
163 [me] = mean(mean_ok_trainval);
|
wolffd@0
|
164 mean_ok_train(j) = struct('max',ma , ...
|
wolffd@0
|
165 'max_idx',valid_ids(maidx) , ...
|
wolffd@0
|
166 'min',mi , ...
|
wolffd@0
|
167 'min_idx',valid_ids(miidx) , ...
|
wolffd@0
|
168 'mean',me);
|
wolffd@0
|
169 end
|
wolffd@0
|
170
|
wolffd@0
|
171 % ---
|
wolffd@0
|
172 % get the statistics over the different values
|
wolffd@0
|
173 % this parameter can hold
|
wolffd@0
|
174 %
|
wolffd@0
|
175 % CAVE/TODO: the idx references are relative to valid_idx
|
wolffd@0
|
176 % ---
|
wolffd@0
|
177 [best, absolute.best_idx] = max([mean_ok_test.max]);
|
wolffd@0
|
178 [worst, absolute.worst_idx] = min([mean_ok_test.max]);
|
wolffd@0
|
179
|
wolffd@0
|
180 % ---
|
wolffd@0
|
181 % get differences:
|
wolffd@0
|
182 difference.max = max([mean_ok_test.max]) - min([mean_ok_test.max]);
|
wolffd@0
|
183
|
wolffd@0
|
184 % format output
|
wolffd@0
|
185 out.entries = entries;
|
wolffd@0
|
186 out.mean_ok_test = mean_ok_test;
|
wolffd@0
|
187 out.mean_ok_train = mean_ok_train;
|
wolffd@0
|
188 out.difference = difference;
|
wolffd@0
|
189 out.absolute = absolute;
|
wolffd@0
|
190 end
|
wolffd@0
|
191
|
wolffd@0
|
192
|
wolffd@0
|
193 % ---
|
wolffd@0
|
194 % display
|
wolffd@0
|
195 % ---
|
wolffd@0
|
196 function a = display_param_influence(stats)
|
wolffd@0
|
197
|
wolffd@0
|
198 if isempty(stats)
|
wolffd@0
|
199 return;
|
wolffd@0
|
200 end
|
wolffd@0
|
201
|
wolffd@0
|
202 ptypes = fieldnames(stats);
|
wolffd@0
|
203
|
wolffd@0
|
204 dmean = [];
|
wolffd@0
|
205 dmax = [];
|
wolffd@0
|
206 best_val = {};
|
wolffd@0
|
207 for i = 1:numel(ptypes)
|
wolffd@0
|
208
|
wolffd@0
|
209 % serialise the statistics
|
wolffd@0
|
210 % dmean = [dmean stats.(ptypes{i}).difference.mean];
|
wolffd@0
|
211 dmax = [dmax stats.(ptypes{i}).difference.max];
|
wolffd@0
|
212 best_val = {best_val{:} stats.(ptypes{i}).entries( ...
|
wolffd@0
|
213 stats.(ptypes{i}).absolute.best_idx) };
|
wolffd@0
|
214
|
wolffd@0
|
215 % take care of string args
|
wolffd@0
|
216 if isnumeric(best_val{i})
|
wolffd@0
|
217 lbl{i} = sprintf('%5.2f' ,best_val{i});
|
wolffd@0
|
218 else
|
wolffd@0
|
219 lbl{i} = best_val{i};
|
wolffd@0
|
220 end
|
wolffd@0
|
221 end
|
wolffd@0
|
222
|
wolffd@0
|
223
|
wolffd@0
|
224 bar([dmax]'* 100);
|
wolffd@0
|
225 colormap(1-spring);
|
wolffd@0
|
226 % legend({'maximal effect on mean correctness'})
|
wolffd@0
|
227 xlabel('effect on max. correctness for best + worst case of other parameters');
|
wolffd@0
|
228 ylabel('correctness (0-100%)');
|
wolffd@0
|
229 a = gca;
|
wolffd@0
|
230 set(a,'XTick', 1:numel(ptypes), ...
|
wolffd@0
|
231 'XTickLabel', ptypes);
|
wolffd@0
|
232
|
wolffd@0
|
233 % display best param results
|
wolffd@0
|
234 for i = 1:numel(ptypes)
|
wolffd@0
|
235 text(i,0,lbl{i}, 'color','k');
|
wolffd@0
|
236 end
|
wolffd@0
|
237
|
wolffd@0
|
238 end
|