wolffd@0: function stats = test_generic_display_param_influence(results, show)
wolffd@0: % returns the mean accuracy influence of each feature and training parameter 
wolffd@0: %
wolffd@0: % the influence is measured by comparing the mean 
wolffd@0: % achieved accuracy for all tries with each specific 
wolffd@0: % parameter being constant
wolffd@0: % 
wolffd@0: % TODO: evaluate how the comparisons of all configuration
wolffd@0: % tuples twith just the specific analysed parameter 
wolffd@0: % changing differ from the above approach
wolffd@0: 
wolffd@0: % get statistics for feature parameters
wolffd@0: stats.fparams = gen_param_influence(results, 'fparams');
wolffd@0: 
wolffd@0: % get statistics for feature parameters
wolffd@0: if isfield(results, 'trainparams')
wolffd@0:     
wolffd@0:     stats.trainparams = gen_param_influence(results, 'trainparams');
wolffd@0:     
wolffd@0:     % the following case is for backwards compability
wolffd@0: elseif isfield(results, 'mlrparams')
wolffd@0:     
wolffd@0:     stats.trainparams = gen_param_influence(results, 'mlrparams');
wolffd@0: end
wolffd@0: 
wolffd@0: if show
wolffd@0:     % display results
wolffd@0: 
wolffd@0:     if ~isempty(stats.fparams)
wolffd@0:         figure;
wolffd@0:     %    subplot(2,1,1);
wolffd@0:         display_param_influence(stats.fparams);
wolffd@0:     end
wolffd@0: 
wolffd@0:     if ~isempty(stats.trainparams)
wolffd@0:         figure;
wolffd@0:     %    subplot(2,1,2);
wolffd@0:         display_param_influence(stats.trainparams);
wolffd@0:     end
wolffd@0: end
wolffd@0: 
wolffd@0: end
wolffd@0: 
wolffd@0: % --- 
wolffd@0: % gen_param_influence
wolffd@0: % ---
wolffd@0: function stats = gen_param_influence(results, paramname)
wolffd@0: % generates statistics given results and parameter type as string. 
wolffd@0: 
wolffd@0: % get individual fields of this parameter set
wolffd@0: ptypes = fieldnames(results(1).(paramname));
wolffd@0: 
wolffd@0: for i = 1:numel(ptypes)
wolffd@0:     % ---
wolffd@0:     % get all individual configurations of this parameter.
wolffd@0:     % ---
wolffd@0:     allvals = [results.(paramname)];
wolffd@0:     
wolffd@0:     % take care of string args
wolffd@0:     if ~ischar(allvals(1).(ptypes{i}))
wolffd@0:         if ~iscell(allvals(1).(ptypes{i}))
wolffd@0:             
wolffd@0:             % parameter array of chars 
wolffd@0:             allvals = [allvals.(ptypes{i})];
wolffd@0:         else
wolffd@0:             % complex parameter array of cells 
wolffd@0:             for j=1:numel(allvals)
wolffd@0:                 tmpvals{j} = cell2str(allvals(j).(ptypes{i}));
wolffd@0:             end
wolffd@0:             allvals = tmpvals;
wolffd@0:         end
wolffd@0:     else
wolffd@0:         % parameter array of numbers 
wolffd@0:         allvals = {allvals.(ptypes{i})};
wolffd@0:     end
wolffd@0:     
wolffd@0:     % save using original parameter name
wolffd@0:     tmp = param_influence(results, allvals);
wolffd@0:     
wolffd@0:     if ~isempty(tmp)
wolffd@0:         stats.(ptypes{i}) = tmp;
wolffd@0:     end   
wolffd@0: end
wolffd@0: 
wolffd@0: if ~exist('stats','var')
wolffd@0:     stats = [];
wolffd@0: end
wolffd@0: 
wolffd@0: end
wolffd@0: 
wolffd@0: 
wolffd@0: % --- 
wolffd@0: % param_influence
wolffd@0: % ---
wolffd@0: function out = param_influence(results, allvals)
wolffd@0: % give the influence (given results) for the parameter settings
wolffd@0: % given in allvals.
wolffd@0: %
wolffd@0: % numel(results) = numel(allvals)
wolffd@0: 
wolffd@0:     % ---
wolffd@0:     % get all different settings of this parameter.
wolffd@0:     % NOTE: this might also work results-of the box for strings.
wolffd@0:     % not tested, below has to be changed ot cell / matrix notations
wolffd@0:     % ---
wolffd@0:     entries = unique(allvals);
wolffd@0:     
wolffd@0:     % just calculate for params with more than one option
wolffd@0:     if numel(entries) < 2 || ischar(entries)
wolffd@0:         
wolffd@0:         out = [];
wolffd@0:         return;
wolffd@0:     end
wolffd@0:     
wolffd@0:     % calculate statstics for this fixed parameter
wolffd@0:     for j = 1:numel(entries)
wolffd@0:         
wolffd@0:         % care for string parameters
wolffd@0:         if ~(iscell(allvals) && ischar(allvals{1}))
wolffd@0:             valid_idx = (allvals == entries(j));
wolffd@0:             
wolffd@0:             % mean_ok_test
wolffd@0:             valid_ids = find(valid_idx);
wolffd@0:         else
wolffd@0:             valid_ids = strcellfind(allvals, entries{j}, 1);
wolffd@0:         end
wolffd@0:         
wolffd@0:         % ---
wolffd@0:         % get the relevant statistics over the variations
wolffd@0:         % of the further parameters
wolffd@0:         % ---
wolffd@0:         mean_ok_testval = [];
wolffd@0:         for i = 1:numel(valid_ids)
wolffd@0:              mean_ok_testval = [mean_ok_testval results(valid_ids(i)).mean_ok_test(1,:)];
wolffd@0:         end
wolffd@0: 
wolffd@0:         [ma,maidx] = max(mean_ok_testval);
wolffd@0:         [mi,miidx] = min(mean_ok_testval);
wolffd@0:         [me] = mean(mean_ok_testval);
wolffd@0:         mean_ok_test(j) = struct('max',ma , ...
wolffd@0:                             'max_idx',valid_ids(maidx) , ...
wolffd@0:                             'min',mi , ...
wolffd@0:                             'min_idx',valid_ids(miidx) , ...
wolffd@0:                             'mean',me);
wolffd@0:                         
wolffd@0:         % ---
wolffd@0:         % get the training statistics over the variations
wolffd@0:         % of the further parameters
wolffd@0:         % ---
wolffd@0:         mean_ok_trainval = [];
wolffd@0:         for i = 1:numel(valid_ids)
wolffd@0:              mean_ok_trainval = [mean_ok_trainval results(valid_ids(i)).mean_ok_train(1,:)];
wolffd@0:         end
wolffd@0: 
wolffd@0:         [ma,maidx] = max(mean_ok_trainval);
wolffd@0:         % ---
wolffd@0:         % NOTE :this allowed for accesment of improvement by RBM selection
wolffd@0: %         warning testing random idx instead of best one 
wolffd@0: %         maidx = max(1, round(rand(1)* numel(valid_ids)));
wolffd@0: %       % ---
wolffd@0: 
wolffd@0:         [mi,miidx] = min(mean_ok_trainval);
wolffd@0:         [me] = mean(mean_ok_trainval);
wolffd@0:         mean_ok_train(j) = struct('max',ma , ...
wolffd@0:                             'max_idx',valid_ids(maidx) , ...
wolffd@0:                             'min',mi , ...
wolffd@0:                             'min_idx',valid_ids(miidx) , ...
wolffd@0:                             'mean',me);
wolffd@0:     end
wolffd@0:     
wolffd@0:     % ---
wolffd@0:     % get the statistics over the different values 
wolffd@0:     % this parameter can hold
wolffd@0:     %
wolffd@0:     % CAVE/TODO: the idx references are relative to valid_idx
wolffd@0:     % ---
wolffd@0:     [best, absolute.best_idx] = max([mean_ok_test.max]);
wolffd@0:     [worst, absolute.worst_idx] = min([mean_ok_test.max]);
wolffd@0:     
wolffd@0:     % ---
wolffd@0:     % get differences:
wolffd@0:     difference.max = max([mean_ok_test.max]) - min([mean_ok_test.max]);
wolffd@0:     
wolffd@0:     % format output
wolffd@0:     out.entries = entries;
wolffd@0:     out.mean_ok_test = mean_ok_test;
wolffd@0:     out.mean_ok_train = mean_ok_train;
wolffd@0:     out.difference = difference;
wolffd@0:     out.absolute = absolute;
wolffd@0: end
wolffd@0:     
wolffd@0: 
wolffd@0: % --- 
wolffd@0: % display
wolffd@0: % ---
wolffd@0: function a = display_param_influence(stats)
wolffd@0: 
wolffd@0: if isempty(stats)
wolffd@0:     return;
wolffd@0: end
wolffd@0: 
wolffd@0: ptypes = fieldnames(stats);
wolffd@0: 
wolffd@0: dmean = [];
wolffd@0: dmax = [];
wolffd@0: best_val = {};
wolffd@0: for i = 1:numel(ptypes)
wolffd@0:     
wolffd@0:     % serialise the statistics
wolffd@0: %    dmean = [dmean stats.(ptypes{i}).difference.mean];
wolffd@0:      dmax = [dmax stats.(ptypes{i}).difference.max];
wolffd@0:     best_val = {best_val{:} stats.(ptypes{i}).entries( ...
wolffd@0:         stats.(ptypes{i}).absolute.best_idx) };
wolffd@0:     
wolffd@0:     % take care of string args
wolffd@0:     if isnumeric(best_val{i})
wolffd@0:         lbl{i} = sprintf('%5.2f' ,best_val{i});
wolffd@0:     else
wolffd@0:         lbl{i} = best_val{i};
wolffd@0:     end
wolffd@0: end
wolffd@0: 
wolffd@0: 
wolffd@0: bar([dmax]'* 100);
wolffd@0: colormap(1-spring);
wolffd@0: % legend({'maximal effect on mean correctness'})
wolffd@0: xlabel('effect on max. correctness for best + worst case of other parameters');
wolffd@0: ylabel('correctness (0-100%)');
wolffd@0: a = gca;
wolffd@0: set(a,'XTick', 1:numel(ptypes), ...
wolffd@0:     'XTickLabel', ptypes);
wolffd@0: 
wolffd@0: % display best param results
wolffd@0: for i = 1:numel(ptypes)
wolffd@0:     text(i,0,lbl{i}, 'color','k');
wolffd@0: end
wolffd@0: 
wolffd@0: end