wolffd@0: function [out, stats] = write_mat_results_ISMIR13RBM_windows(dirin,fileout)
wolffd@0: % combine the test results from the directories supplied,
wolffd@0: % and recombine the bins to a coherent dataset
wolffd@0: 
wolffd@0: % folders_win1 = {'130507_svm_disttest_subsconv_rbm_largegrid_21_r2385','130507_svm_disttest_subsconv_rbm_largegrid_22_r2385','130507_svm_disttest_subsconv_rbm_largegrid_23_r2385','130507_svm_disttest_subsconv_rbm_largegrid_24_r2385','130508_svm_disttest_subsconv_rbm_largegrid_25_r2391'};
wolffd@0: % folders_win2 = {'130511_svm_disttest_subsconv_rbm_largegrid_21_r2391','130511_svm_disttest_subsconv_rbm_largegrid_22_r2391','130511_svm_disttest_subsconv_rbm_largegrid_23_r2391','130511_svm_disttest_subsconv_rbm_largegrid_24_r2391','130513_svm_disttest_subsconv_rbm_largegrid_25_r2391'};
wolffd@0: %folders = {folders_win1{:}, folders_win2{:}};
wolffd@0: 
wolffd@0: features = [];
wolffd@0: show = 1;
wolffd@0: 
wolffd@0: if nargin == 0
wolffd@0:     dirin{1} = './';
wolffd@0: end
wolffd@0: 
wolffd@0: global comparison;
wolffd@0: global comparison_ids;
wolffd@0:      
wolffd@0: newout = [];
wolffd@0: thisdir = pwd;
wolffd@0: % loop through al lthe result directories and 
wolffd@0: for diri = 1:numel(dirin)
wolffd@0: 
wolffd@0:     % ---
wolffd@0:     % go to directory and locate file
wolffd@0:     % ---
wolffd@0:     cd(dirin{diri});
wolffd@0: 
wolffd@0:     u = dir();
wolffd@0:     u = {u.name};
wolffd@0:     [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1);
wolffd@0:     
wolffd@0:     if numel(idx) < 1
wolffd@0:         error 'This directory contains no valid test data';
wolffd@0:     end
wolffd@0:     
wolffd@0:     % just one or more tests in this folder?
wolffd@0:     if exist('file','var') && isnumeric(file)
wolffd@0:         cprint(1, 'loading one result file');
wolffd@0:         file = u{idx(file)};
wolffd@0:         data = load(file);
wolffd@0:         sappend(out,data.out);
wolffd@0:     else
wolffd@0:         for filei = 1:numel(idx)
wolffd@0:             cprint(1, 'loading result file %i of %i',filei, numel(idx));
wolffd@0:             file = u{idx(filei)};
wolffd@0:             data = load(file);
wolffd@0:             newout = sappend(newout,data.out);
wolffd@0:         end
wolffd@0:     end
wolffd@0:     % reset act directory
wolffd@0:     cd(thisdir);
wolffd@0: end
wolffd@0: 
wolffd@0: % ---
wolffd@0: % filter C values!
wolffd@0: % ---
wolffd@0: allcs = zeros(numel(newout),1);
wolffd@0: for i=1:numel(newout)
wolffd@0:     allcs(i) = newout(i).trainparams.C;
wolffd@0: end
wolffd@0: valididx = find(allcs == 1); % select C!
wolffd@0: newout = newout(valididx);
wolffd@0: 
wolffd@0: % ---
wolffd@0: % filter according to training parameter deltafun_params(windows)
wolffd@0: % ---
wolffd@0: wind = cell(numel(newout),1);
wolffd@0: for i=1:numel(newout)
wolffd@0:     wind{i} = cell2str(newout(i).trainparams.deltafun_params);
wolffd@0: end
wolffd@0: windu = unique(wind);
wolffd@0: 
wolffd@0: % save the combined out
wolffd@0: newout2 = newout;
wolffd@0: 
wolffd@0: % do the loop as in write_mat_results_ISMIR13RBM
wolffd@0: for wi=1:numel(windu)
wolffd@0:     newout = newout2(strcellfind(wind, windu{wi},1));
wolffd@0:     
wolffd@0:     % bundle all datasets
wolffd@0:     fout = sameparamsubset(newout, 'dataset','');
wolffd@0:     out = [];
wolffd@0:     for ci=1:numel(fout)
wolffd@0:         filteredout = fout{ci};
wolffd@0: 
wolffd@0:         ok_test = zeros(2, numel(filteredout));
wolffd@0:         ok_train =  zeros(2, numel(filteredout));
wolffd@0:         ok_config = [];
wolffd@0: 
wolffd@0: 
wolffd@0:         tmpout = filteredout(1);
wolffd@0:         % cycle over all test sets and get new means
wolffd@0:         for i=1:numel(filteredout)
wolffd@0:             ok_test(:,i) = filteredout(i).mean_ok_test;
wolffd@0:             ok_train(:,i) = filteredout(i).mean_ok_train;
wolffd@0:         end
wolffd@0: 
wolffd@0:         % save the stuff
wolffd@0:         tmpout.mean_ok_test = mean(ok_test,2);
wolffd@0:         tmpout.var_ok_test = var(ok_test,0,2);
wolffd@0:         tmpout.mean_ok_train = mean(ok_train,2);
wolffd@0:         tmpout.var_ok_train = var(ok_train,0,2);
wolffd@0: 
wolffd@0:         tmpout.ok_test = ok_test;
wolffd@0:         tmpout.ok_train = ok_train;
wolffd@0: 
wolffd@0:         % put it in output structure
wolffd@0:         out = sappend(out,tmpout);
wolffd@0:     end
wolffd@0: 
wolffd@0: %     % ---
wolffd@0: %     % show results
wolffd@0: %     % ---
wolffd@0: %     if numel([out.mean_ok_test]) > 1 && show
wolffd@0: % 
wolffd@0: %         % plot means  % plot std = sqrt(var) % plot training results
wolffd@0: %         figure;
wolffd@0: %         boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]);
wolffd@0: %         title (sprintf('Performance for all configs'));
wolffd@0: %     end
wolffd@0: 
wolffd@0:     % --- 
wolffd@0:     % write max. test/train success
wolffd@0:     % ---
wolffd@0:     mean_ok_train = [out.mean_ok_train];
wolffd@0:     [val, idx] = max(mean_ok_train(1,:));
wolffd@0:     if show
wolffd@0:         fprintf(' --- Maximal train set success for %s: nr. %d, %3.2f percent, test result %3.2f percent. --- \n', windu{wi}, idx, val * 100,out(idx).mean_ok_test(1,:)*100)
wolffd@0:     end
wolffd@0: 
wolffd@0: end
wolffd@0: 
wolffd@0: end
wolffd@0: 
wolffd@0: function [out, param_hash,idx] = sameparamsubset(in, ignoret,ignoref)
wolffd@0:     % ---
wolffd@0:     % build index of all existing configurations
wolffd@0:     % ---
wolffd@0:     param_hash = cell(numel(in),1);
wolffd@0:     for i=1:numel(in)
wolffd@0:         params = struct('trainparams',in(i).trainparams, ...
wolffd@0:                         'fparams',in(i).fparams);
wolffd@0: 
wolffd@0:         % remove the dataset param
wolffd@0:         if ~isempty(ignoret)
wolffd@0:             params.trainparams = rmfield(params.trainparams,ignoret);
wolffd@0:         end
wolffd@0:         if ~isempty(ignoref)
wolffd@0:             params.fparams = rmfield(params.fparams,ignoref);
wolffd@0:         end
wolffd@0: 
wolffd@0:         phash = hash(xml_format(params),'md5');          
wolffd@0:         param_hash{i} = phash;
wolffd@0:     end
wolffd@0: 
wolffd@0:     % ---
wolffd@0:     %  recombine the data for different datasets!
wolffd@0:     % ---
wolffd@0:     cvals = unique(param_hash);
wolffd@0: 
wolffd@0:     out = {};
wolffd@0:     for ci=1:numel(cvals)
wolffd@0:         idx{ci} = strcellfind(param_hash,cvals(ci),1);
wolffd@0:         out{ci} = in(idx{ci});
wolffd@0:     end
wolffd@0: 
wolffd@0: end
wolffd@0: 
wolffd@0: function boxplot(mean, std, train);
wolffd@0: 
wolffd@0:     bar([train; mean]', 1.5);
wolffd@0:     hold on;
wolffd@0:     errorbar(1:size(mean,2), mean(1,:), std(1,:),'.');
wolffd@0: %     plot(train,'rO');
wolffd@0:     colormap(spring);
wolffd@0:     axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]);
wolffd@0: end