view core/magnatagatune/tests_evals/test_generic_display_param_influence.m @ 0:e9a9cd732c1e tip

first hg version after svn

function stats = test_generic_display_param_influence(results, show)
% returns the mean accuracy influence of each feature and training parameter 
%
% the influence is measured by comparing the mean 
% achieved accuracy for all tries with each specific 
% parameter being constant
% 
% TODO: evaluate how the comparisons of all configuration
% tuples twith just the specific analysed parameter 
% changing differ from the above approach

% get statistics for feature parameters
stats.fparams = gen_param_influence(results, 'fparams');

% get statistics for feature parameters
if isfield(results, 'trainparams')
    
    stats.trainparams = gen_param_influence(results, 'trainparams');
    
    % the following case is for backwards compability
elseif isfield(results, 'mlrparams')
    
    stats.trainparams = gen_param_influence(results, 'mlrparams');
end

if show
    % display results

    if ~isempty(stats.fparams)
        figure;
    %    subplot(2,1,1);
        display_param_influence(stats.fparams);
    end

    if ~isempty(stats.trainparams)
        figure;
    %    subplot(2,1,2);
        display_param_influence(stats.trainparams);
    end
end

end

% --- 
% gen_param_influence
% ---
function stats = gen_param_influence(results, paramname)
% generates statistics given results and parameter type as string. 

% get individual fields of this parameter set
ptypes = fieldnames(results(1).(paramname));

for i = 1:numel(ptypes)
    % ---
    % get all individual configurations of this parameter.
    % ---
    allvals = [results.(paramname)];
    
    % take care of string args
    if ~ischar(allvals(1).(ptypes{i}))
        if ~iscell(allvals(1).(ptypes{i}))
            
            % parameter array of chars 
            allvals = [allvals.(ptypes{i})];
        else
            % complex parameter array of cells 
            for j=1:numel(allvals)
                tmpvals{j} = cell2str(allvals(j).(ptypes{i}));
            end
            allvals = tmpvals;
        end
    else
        % parameter array of numbers 
        allvals = {allvals.(ptypes{i})};
    end
    
    % save using original parameter name
    tmp = param_influence(results, allvals);
    
    if ~isempty(tmp)
        stats.(ptypes{i}) = tmp;
    end   
end

if ~exist('stats','var')
    stats = [];
end

end


% --- 
% param_influence
% ---
function out = param_influence(results, allvals)
% give the influence (given results) for the parameter settings
% given in allvals.
%
% numel(results) = numel(allvals)

    % ---
    % get all different settings of this parameter.
    % NOTE: this might also work results-of the box for strings.
    % not tested, below has to be changed ot cell / matrix notations
    % ---
    entries = unique(allvals);
    
    % just calculate for params with more than one option
    if numel(entries) < 2 || ischar(entries)
        
        out = [];
        return;
    end
    
    % calculate statstics for this fixed parameter
    for j = 1:numel(entries)
        
        % care for string parameters
        if ~(iscell(allvals) && ischar(allvals{1}))
            valid_idx = (allvals == entries(j));
            
            % mean_ok_test
            valid_ids = find(valid_idx);
        else
            valid_ids = strcellfind(allvals, entries{j}, 1);
        end
        
        % ---
        % get the relevant statistics over the variations
        % of the further parameters
        % ---
        mean_ok_testval = [];
        for i = 1:numel(valid_ids)
             mean_ok_testval = [mean_ok_testval results(valid_ids(i)).mean_ok_test(1,:)];
        end

        [ma,maidx] = max(mean_ok_testval);
        [mi,miidx] = min(mean_ok_testval);
        [me] = mean(mean_ok_testval);
        mean_ok_test(j) = struct('max',ma , ...
                            'max_idx',valid_ids(maidx) , ...
                            'min',mi , ...
                            'min_idx',valid_ids(miidx) , ...
                            'mean',me);
                        
        % ---
        % get the training statistics over the variations
        % of the further parameters
        % ---
        mean_ok_trainval = [];
        for i = 1:numel(valid_ids)
             mean_ok_trainval = [mean_ok_trainval results(valid_ids(i)).mean_ok_train(1,:)];
        end

        [ma,maidx] = max(mean_ok_trainval);
        % ---
        % NOTE :this allowed for accesment of improvement by RBM selection
%         warning testing random idx instead of best one 
%         maidx = max(1, round(rand(1)* numel(valid_ids)));
%       % ---

        [mi,miidx] = min(mean_ok_trainval);
        [me] = mean(mean_ok_trainval);
        mean_ok_train(j) = struct('max',ma , ...
                            'max_idx',valid_ids(maidx) , ...
                            'min',mi , ...
                            'min_idx',valid_ids(miidx) , ...
                            'mean',me);
    end
    
    % ---
    % get the statistics over the different values 
    % this parameter can hold
    %
    % CAVE/TODO: the idx references are relative to valid_idx
    % ---
    [best, absolute.best_idx] = max([mean_ok_test.max]);
    [worst, absolute.worst_idx] = min([mean_ok_test.max]);
    
    % ---
    % get differences:
    difference.max = max([mean_ok_test.max]) - min([mean_ok_test.max]);
    
    % format output
    out.entries = entries;
    out.mean_ok_test = mean_ok_test;
    out.mean_ok_train = mean_ok_train;
    out.difference = difference;
    out.absolute = absolute;
end
    

% --- 
% display
% ---
function a = display_param_influence(stats)

if isempty(stats)
    return;
end

ptypes = fieldnames(stats);

dmean = [];
dmax = [];
best_val = {};
for i = 1:numel(ptypes)
    
    % serialise the statistics
%    dmean = [dmean stats.(ptypes{i}).difference.mean];
     dmax = [dmax stats.(ptypes{i}).difference.max];
    best_val = {best_val{:} stats.(ptypes{i}).entries( ...
        stats.(ptypes{i}).absolute.best_idx) };
    
    % take care of string args
    if isnumeric(best_val{i})
        lbl{i} = sprintf('%5.2f' ,best_val{i});
    else
        lbl{i} = best_val{i};
    end
end


bar([dmax]'* 100);
colormap(1-spring);
% legend({'maximal effect on mean correctness'})
xlabel('effect on max. correctness for best + worst case of other parameters');
ylabel('correctness (0-100%)');
a = gca;
set(a,'XTick', 1:numel(ptypes), ...
    'XTickLabel', ptypes);

% display best param results
for i = 1:numel(ptypes)
    text(i,0,lbl{i}, 'color','k');
end

end