Mercurial > hg > camir-aes2014

annotate core/magnatagatune/tests_evals/test_generic_display_param_influence.m @ 0:e9a9cd732c1e tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first hg version after svn
author wolffd
date Tue, 10 Feb 2015 15:05:51 +0000
parents
children
rev   line source
wolffd@0 1 function stats = test_generic_display_param_influence(results, show)
wolffd@0 2 % returns the mean accuracy influence of each feature and training parameter
wolffd@0 3 %
wolffd@0 4 % the influence is measured by comparing the mean
wolffd@0 5 % achieved accuracy for all tries with each specific
wolffd@0 6 % parameter being constant
wolffd@0 7 %
wolffd@0 8 % TODO: evaluate how the comparisons of all configuration
wolffd@0 9 % tuples twith just the specific analysed parameter
wolffd@0 10 % changing differ from the above approach
wolffd@0 11
wolffd@0 12 % get statistics for feature parameters
wolffd@0 13 stats.fparams = gen_param_influence(results, 'fparams');
wolffd@0 14
wolffd@0 15 % get statistics for feature parameters
wolffd@0 16 if isfield(results, 'trainparams')
wolffd@0 17
wolffd@0 18 stats.trainparams = gen_param_influence(results, 'trainparams');
wolffd@0 19
wolffd@0 20 % the following case is for backwards compability
wolffd@0 21 elseif isfield(results, 'mlrparams')
wolffd@0 22
wolffd@0 23 stats.trainparams = gen_param_influence(results, 'mlrparams');
wolffd@0 24 end
wolffd@0 25
wolffd@0 26 if show
wolffd@0 27 % display results
wolffd@0 28
wolffd@0 29 if ~isempty(stats.fparams)
wolffd@0 30 figure;
wolffd@0 31 % subplot(2,1,1);
wolffd@0 32 display_param_influence(stats.fparams);
wolffd@0 33 end
wolffd@0 34
wolffd@0 35 if ~isempty(stats.trainparams)
wolffd@0 36 figure;
wolffd@0 37 % subplot(2,1,2);
wolffd@0 38 display_param_influence(stats.trainparams);
wolffd@0 39 end
wolffd@0 40 end
wolffd@0 41
wolffd@0 42 end
wolffd@0 43
wolffd@0 44 % ---
wolffd@0 45 % gen_param_influence
wolffd@0 46 % ---
wolffd@0 47 function stats = gen_param_influence(results, paramname)
wolffd@0 48 % generates statistics given results and parameter type as string.
wolffd@0 49
wolffd@0 50 % get individual fields of this parameter set
wolffd@0 51 ptypes = fieldnames(results(1).(paramname));
wolffd@0 52
wolffd@0 53 for i = 1:numel(ptypes)
wolffd@0 54 % ---
wolffd@0 55 % get all individual configurations of this parameter.
wolffd@0 56 % ---
wolffd@0 57 allvals = [results.(paramname)];
wolffd@0 58
wolffd@0 59 % take care of string args
wolffd@0 60 if ~ischar(allvals(1).(ptypes{i}))
wolffd@0 61 if ~iscell(allvals(1).(ptypes{i}))
wolffd@0 62
wolffd@0 63 % parameter array of chars
wolffd@0 64 allvals = [allvals.(ptypes{i})];
wolffd@0 65 else
wolffd@0 66 % complex parameter array of cells
wolffd@0 67 for j=1:numel(allvals)
wolffd@0 68 tmpvals{j} = cell2str(allvals(j).(ptypes{i}));
wolffd@0 69 end
wolffd@0 70 allvals = tmpvals;
wolffd@0 71 end
wolffd@0 72 else
wolffd@0 73 % parameter array of numbers
wolffd@0 74 allvals = {allvals.(ptypes{i})};
wolffd@0 75 end
wolffd@0 76
wolffd@0 77 % save using original parameter name
wolffd@0 78 tmp = param_influence(results, allvals);
wolffd@0 79
wolffd@0 80 if ~isempty(tmp)
wolffd@0 81 stats.(ptypes{i}) = tmp;
wolffd@0 82 end
wolffd@0 83 end
wolffd@0 84
wolffd@0 85 if ~exist('stats','var')
wolffd@0 86 stats = [];
wolffd@0 87 end
wolffd@0 88
wolffd@0 89 end
wolffd@0 90
wolffd@0 91
wolffd@0 92 % ---
wolffd@0 93 % param_influence
wolffd@0 94 % ---
wolffd@0 95 function out = param_influence(results, allvals)
wolffd@0 96 % give the influence (given results) for the parameter settings
wolffd@0 97 % given in allvals.
wolffd@0 98 %
wolffd@0 99 % numel(results) = numel(allvals)
wolffd@0 100
wolffd@0 101 % ---
wolffd@0 102 % get all different settings of this parameter.
wolffd@0 103 % NOTE: this might also work results-of the box for strings.
wolffd@0 104 % not tested, below has to be changed ot cell / matrix notations
wolffd@0 105 % ---
wolffd@0 106 entries = unique(allvals);
wolffd@0 107
wolffd@0 108 % just calculate for params with more than one option
wolffd@0 109 if numel(entries) < 2 || ischar(entries)
wolffd@0 110
wolffd@0 111 out = [];
wolffd@0 112 return;
wolffd@0 113 end
wolffd@0 114
wolffd@0 115 % calculate statstics for this fixed parameter
wolffd@0 116 for j = 1:numel(entries)
wolffd@0 117
wolffd@0 118 % care for string parameters
wolffd@0 119 if ~(iscell(allvals) && ischar(allvals{1}))
wolffd@0 120 valid_idx = (allvals == entries(j));
wolffd@0 121
wolffd@0 122 % mean_ok_test
wolffd@0 123 valid_ids = find(valid_idx);
wolffd@0 124 else
wolffd@0 125 valid_ids = strcellfind(allvals, entries{j}, 1);
wolffd@0 126 end
wolffd@0 127
wolffd@0 128 % ---
wolffd@0 129 % get the relevant statistics over the variations
wolffd@0 130 % of the further parameters
wolffd@0 131 % ---
wolffd@0 132 mean_ok_testval = [];
wolffd@0 133 for i = 1:numel(valid_ids)
wolffd@0 134 mean_ok_testval = [mean_ok_testval results(valid_ids(i)).mean_ok_test(1,:)];
wolffd@0 135 end
wolffd@0 136
wolffd@0 137 [ma,maidx] = max(mean_ok_testval);
wolffd@0 138 [mi,miidx] = min(mean_ok_testval);
wolffd@0 139 [me] = mean(mean_ok_testval);
wolffd@0 140 mean_ok_test(j) = struct('max',ma , ...
wolffd@0 141 'max_idx',valid_ids(maidx) , ...
wolffd@0 142 'min',mi , ...
wolffd@0 143 'min_idx',valid_ids(miidx) , ...
wolffd@0 144 'mean',me);
wolffd@0 145
wolffd@0 146 % ---
wolffd@0 147 % get the training statistics over the variations
wolffd@0 148 % of the further parameters
wolffd@0 149 % ---
wolffd@0 150 mean_ok_trainval = [];
wolffd@0 151 for i = 1:numel(valid_ids)
wolffd@0 152 mean_ok_trainval = [mean_ok_trainval results(valid_ids(i)).mean_ok_train(1,:)];
wolffd@0 153 end
wolffd@0 154
wolffd@0 155 [ma,maidx] = max(mean_ok_trainval);
wolffd@0 156 % ---
wolffd@0 157 % NOTE :this allowed for accesment of improvement by RBM selection
wolffd@0 158 % warning testing random idx instead of best one
wolffd@0 159 % maidx = max(1, round(rand(1)* numel(valid_ids)));
wolffd@0 160 % % ---
wolffd@0 161
wolffd@0 162 [mi,miidx] = min(mean_ok_trainval);
wolffd@0 163 [me] = mean(mean_ok_trainval);
wolffd@0 164 mean_ok_train(j) = struct('max',ma , ...
wolffd@0 165 'max_idx',valid_ids(maidx) , ...
wolffd@0 166 'min',mi , ...
wolffd@0 167 'min_idx',valid_ids(miidx) , ...
wolffd@0 168 'mean',me);
wolffd@0 169 end
wolffd@0 170
wolffd@0 171 % ---
wolffd@0 172 % get the statistics over the different values
wolffd@0 173 % this parameter can hold
wolffd@0 174 %
wolffd@0 175 % CAVE/TODO: the idx references are relative to valid_idx
wolffd@0 176 % ---
wolffd@0 177 [best, absolute.best_idx] = max([mean_ok_test.max]);
wolffd@0 178 [worst, absolute.worst_idx] = min([mean_ok_test.max]);
wolffd@0 179
wolffd@0 180 % ---
wolffd@0 181 % get differences:
wolffd@0 182 difference.max = max([mean_ok_test.max]) - min([mean_ok_test.max]);
wolffd@0 183
wolffd@0 184 % format output
wolffd@0 185 out.entries = entries;
wolffd@0 186 out.mean_ok_test = mean_ok_test;
wolffd@0 187 out.mean_ok_train = mean_ok_train;
wolffd@0 188 out.difference = difference;
wolffd@0 189 out.absolute = absolute;
wolffd@0 190 end
wolffd@0 191
wolffd@0 192
wolffd@0 193 % ---
wolffd@0 194 % display
wolffd@0 195 % ---
wolffd@0 196 function a = display_param_influence(stats)
wolffd@0 197
wolffd@0 198 if isempty(stats)
wolffd@0 199 return;
wolffd@0 200 end
wolffd@0 201
wolffd@0 202 ptypes = fieldnames(stats);
wolffd@0 203
wolffd@0 204 dmean = [];
wolffd@0 205 dmax = [];
wolffd@0 206 best_val = {};
wolffd@0 207 for i = 1:numel(ptypes)
wolffd@0 208
wolffd@0 209 % serialise the statistics
wolffd@0 210 % dmean = [dmean stats.(ptypes{i}).difference.mean];
wolffd@0 211 dmax = [dmax stats.(ptypes{i}).difference.max];
wolffd@0 212 best_val = {best_val{:} stats.(ptypes{i}).entries( ...
wolffd@0 213 stats.(ptypes{i}).absolute.best_idx) };
wolffd@0 214
wolffd@0 215 % take care of string args
wolffd@0 216 if isnumeric(best_val{i})
wolffd@0 217 lbl{i} = sprintf('%5.2f' ,best_val{i});
wolffd@0 218 else
wolffd@0 219 lbl{i} = best_val{i};
wolffd@0 220 end
wolffd@0 221 end
wolffd@0 222
wolffd@0 223
wolffd@0 224 bar([dmax]'* 100);
wolffd@0 225 colormap(1-spring);
wolffd@0 226 % legend({'maximal effect on mean correctness'})
wolffd@0 227 xlabel('effect on max. correctness for best + worst case of other parameters');
wolffd@0 228 ylabel('correctness (0-100%)');
wolffd@0 229 a = gca;
wolffd@0 230 set(a,'XTick', 1:numel(ptypes), ...
wolffd@0 231 'XTickLabel', ptypes);
wolffd@0 232
wolffd@0 233 % display best param results
wolffd@0 234 for i = 1:numel(ptypes)
wolffd@0 235 text(i,0,lbl{i}, 'color','k');
wolffd@0 236 end
wolffd@0 237
wolffd@0 238 end