view core/magnatagatune/tests_evals/rbm_subspace/write_mat_results_ISMIR13RBM.m @ 0:e9a9cd732c1e tip

first hg version after svn

function [out, stats] = write_mat_results_ISMIR13RBM(dirin,fileout)
% combine the test results from the directories supplied,
% and recombine the bins to a coherent dataset
%
% allows for distinguishing different hidden State numbers

% folders = {'130505_svm_disttest_subsconv_rbm_largegrid_21_r2385','130505_svm_disttest_subsconv_rbm_largegrid_22_r2385','130505_svm_disttest_subsconv_rbm_largegrid_23_r2385','130505_svm_disttest_subsconv_rbm_largegrid_24_r2385','130506_svm_disttest_subsconv_rbm_largegrid_25_r2385'};



features = [];
show = 1;

if nargin == 0
    dirin{1} = './';
end

global comparison;
global comparison_ids;
     
newout = [];
thisdir = pwd;
% loop through al lthe result directories and 
for diri = 1:numel(dirin)

    % ---
    % go to directory and locate file
    % ---
    cd(dirin{diri});

    u = dir();
    u = {u.name};
    [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1);
    
    if numel(idx) < 1
        error 'This directory contains no valid test data';
    end
    
    % just one or more tests in this folder?
    if exist('file','var') && isnumeric(file)
        cprint(1, 'loading one result file');
        file = u{idx(file)};
        data = load(file);
        sappend(out,data.out);
    else
        for filei = 1:numel(idx)
            cprint(1, 'loading result file %i of %i',filei, numel(idx));
            file = u{idx(filei)};
            data = load(file);
            newout = sappend(newout,data.out);
        end
    end
    % reset act directory
    cd(thisdir);
end


% ---
% filter C values!
% ---
allcs = zeros(numel(newout),1);
for i=1:numel(newout)
    allcs(i) = newout(i).trainparams.C;
end
valididx = find(allcs == 1); % select C!
newout = newout(valididx);


% bundle all datasets
fout = sameparamsubset(newout, 'dataset','');
out = [];
for ci=1:numel(fout)
    filteredout = fout{ci};
    
    
    ok_test = zeros(2, numel(filteredout));
    ok_train =  zeros(2, numel(filteredout));
    ok_config = [];
    
    
    tmpout = filteredout(1);
    % cycle over all test sets and get new means
    for i=1:numel(filteredout)
        ok_test(:,i) = filteredout(i).mean_ok_test;
        ok_train(:,i) = filteredout(i).mean_ok_train;
    end
    
    % save the stuff
    tmpout.mean_ok_test = mean(ok_test,2);
    tmpout.var_ok_test = var(ok_test,0,2);
    tmpout.mean_ok_train = mean(ok_train,2);
    tmpout.var_ok_train = var(ok_train,0,2);
    
    tmpout.ok_test = ok_test;
    tmpout.ok_train = ok_train;
    
    % put it in output structure
    out = sappend(out,tmpout);
end

% ---
% show results
% ---
if numel([out.mean_ok_test]) > 1 && show
    
    % plot means  % plot std = sqrt(var) % plot training results
    figure;
    boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]);
    title (sprintf('Performance for all configs'));
end



% --- 
% write max. test success
% ---
mean_ok_test = [out.mean_ok_test];
[val, idx] = max(mean_ok_test(1,:));
if show
    fprintf(' --- Maximal test set success: nr. %d, %3.2f percent.  train result %3.2f percent. --- \n', idx, val * 100,out(idx).mean_ok_train(1,:)*100)
end


% --- 
% write max. test/train success
% ---
mean_ok_train = [out.mean_ok_train];
[val, idx] = max(mean_ok_train(1,:));
if show
    fprintf(' --- Maximal train set success: nr. %d, %3.2f percent, test result %3.2f percent. --- \n', idx, val * 100,out(idx).mean_ok_test(1,:)*100)
end

% save this summary
save([hash(strcat(dirin{:}),'md5') '_summary'], 'out');



% ---
% build index of these feature configurations without  num_hid
% ---
[fout2,param_hash,idxhash] = sameparamsubset(out, '','rbm_hidNum');
for ci = 1:numel(fout2)
    
    % search for the training param index
    found = find(idxhash{ci}== idx);
    if ~isempty(found)
        out_withsameconfig = fout2{ci};
        break;
    end
end


end

function [out, param_hash,idx] = sameparamsubset(in, ignoret,ignoref)
    % ---
    % build index of all existing configurations
    % ---
    param_hash = cell(numel(in),1);
    for i=1:numel(in)
        params = struct('trainparams',in(i).trainparams, ...
                        'fparams',in(i).fparams);

        % remove the dataset param
        if ~isempty(ignoret)
            params.trainparams = rmfield(params.trainparams,ignoret);
        end
        if ~isempty(ignoref)
            params.fparams = rmfield(params.fparams,ignoref);
        end

        phash = hash(xml_format(params),'md5');          
        param_hash{i} = phash;
    end

    % ---
    %  recombine the data for different datasets!
    % ---
    cvals = unique(param_hash);

    out = {};
    for ci=1:numel(cvals)
        idx{ci} = strcellfind(param_hash,cvals(ci),1);
        out{ci} = in(idx{ci});
    end

end

function boxplot(mean, std, train);

    bar([train; mean]', 1.5);
    hold on;
    errorbar(1:size(mean,2), mean(1,:), std(1,:),'.');
%     plot(train,'rO');
    colormap(spring);
    axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]);
end