Mercurial > hg > camir-aes2014

annotate core/magnatagatune/tests_evals/rbm_subspace/write_mat_results_ISMIR13RBM_singletraining.m @ 0:e9a9cd732c1e tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first hg version after svn
author wolffd
date Tue, 10 Feb 2015 15:05:51 +0000
parents
children
rev   line source
wolffd@0 1 function [out, stats] = write_mat_results_ISMIR13RBM_singletraining(dirin,fileout)
wolffd@0 2 % combine the test results from the directories supplied,
wolffd@0 3 % group them according to dataset parameter values
wolffd@0 4 % combine the test results from the directories supplied,
wolffd@0 5 % group them according to dataset parameter values
wolffd@0 6
wolffd@0 7 features = [];
wolffd@0 8 show = 1;
wolffd@0 9
wolffd@0 10 if nargin == 0
wolffd@0 11 dirin{1} = './';
wolffd@0 12 end
wolffd@0 13
wolffd@0 14 global comparison;
wolffd@0 15 global comparison_ids;
wolffd@0 16
wolffd@0 17 newout = [];
wolffd@0 18 thisdir = pwd;
wolffd@0 19 % loop through al lthe result directories and
wolffd@0 20 for diri = 1:numel(dirin)
wolffd@0 21
wolffd@0 22 % ---
wolffd@0 23 % go to directory and locate file
wolffd@0 24 % ---
wolffd@0 25 cd(dirin{diri});
wolffd@0 26
wolffd@0 27 u = dir();
wolffd@0 28 u = {u.name};
wolffd@0 29 [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1);
wolffd@0 30
wolffd@0 31 if numel(idx) < 1
wolffd@0 32 error 'This directory contains no valid test data';
wolffd@0 33 end
wolffd@0 34
wolffd@0 35 % just one or more tests in this folder?
wolffd@0 36 if exist('file','var') && isnumeric(file)
wolffd@0 37 cprint(1, 'loading one result file');
wolffd@0 38 file = u{idx(file)};
wolffd@0 39 data = load(file);
wolffd@0 40 sappend(out,data.out);
wolffd@0 41 else
wolffd@0 42 for filei = 1:numel(idx)
wolffd@0 43 cprint(1, 'loading result file %i of %i',filei, numel(idx));
wolffd@0 44 file = u{idx(filei)};
wolffd@0 45 data = load(file);
wolffd@0 46 newout = sappend(newout,data.out);
wolffd@0 47 end
wolffd@0 48 end
wolffd@0 49 % reset act directory
wolffd@0 50 cd(thisdir);
wolffd@0 51 end
wolffd@0 52
wolffd@0 53 % ---
wolffd@0 54 % filter according to training parameter C
wolffd@0 55 %
wolffd@0 56 % NOTE :if we don't filter by C, we get strong overfitting with training
wolffd@0 57 % success > 96 % and test set performance aorund 65 %
wolffd@0 58 % ---
wolffd@0 59 cs = zeros(numel(newout),1);
wolffd@0 60 for i=1:numel(newout)
wolffd@0 61 cs(i) = newout(i).trainparams.C;
wolffd@0 62 end
wolffd@0 63 cvals = unique(cs);
wolffd@0 64
wolffd@0 65 for ci=1:numel(cvals)
wolffd@0 66 valididx = find(cs == cvals(ci));
wolffd@0 67 filteredout = newout(valididx);
wolffd@0 68
wolffd@0 69 % ---
wolffd@0 70 % get parameter statistics
wolffd@0 71 % ---
wolffd@0 72 stats = test_generic_display_param_influence(filteredout, show);
wolffd@0 73
wolffd@0 74 % get maximal values for each test set bin
wolffd@0 75 % ---
wolffd@0 76 % trainparams.dataset contains sets which have each only one bin of the
wolffd@0 77 % ismir testsets
wolffd@0 78 % ---
wolffd@0 79 max_idx = [stats.trainparams.dataset.mean_ok_train.max_idx];
wolffd@0 80 ok_test = zeros(2, numel(max_idx));
wolffd@0 81 ok_train = zeros(2, numel(max_idx));
wolffd@0 82 ok_config = [];
wolffd@0 83 % cycle over all test sets and save best result
wolffd@0 84 for i=1:numel(max_idx)
wolffd@0 85 ok_test(:,i) = filteredout(max_idx(i)).mean_ok_test;
wolffd@0 86 ok_train(:,i) = filteredout(max_idx(i)).mean_ok_train;
wolffd@0 87 ok_config = sappend(ok_config,struct('trainparams',filteredout(max_idx(i)).trainparams, ...
wolffd@0 88 'fparams',filteredout(max_idx(i)).fparams));
wolffd@0 89 end
wolffd@0 90 % save the stuff
wolffd@0 91 out(ci).mean_ok_test = mean(ok_test,2);
wolffd@0 92 out(ci).var_ok_test = var(ok_test,0,2);
wolffd@0 93 out(ci).mean_ok_train = mean(ok_train,2);
wolffd@0 94 out(ci).var_ok_train = var(ok_train,0,2);
wolffd@0 95 out(ci).trainparams.C = cvals(ci);
wolffd@0 96 out(ci).ok_config = ok_config;
wolffd@0 97 out(ci).ok_test = ok_test;
wolffd@0 98 out(ci).ok_train = ok_train;
wolffd@0 99 end
wolffd@0 100
wolffd@0 101 % ---
wolffd@0 102 % show results for different C
wolffd@0 103 % ---
wolffd@0 104 if numel([out.mean_ok_test]) > 1 && show
wolffd@0 105
wolffd@0 106 % plot means % plot std = sqrt(var) % plot training results
wolffd@0 107 figure;
wolffd@0 108 boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]);
wolffd@0 109 title (sprintf('Performance for all configs'));
wolffd@0 110 end
wolffd@0 111
wolffd@0 112 % ---
wolffd@0 113 % write max. test success
wolffd@0 114 % ---
wolffd@0 115 mean_ok_test = [out.mean_ok_test];
wolffd@0 116 [val, idx] = max(mean_ok_test(1,:));
wolffd@0 117 if show
wolffd@0 118 fprintf(' --- Maximal test set success: nr. %d, %3.2f percent. --- \n', idx, val * 100)
wolffd@0 119 end
wolffd@0 120
wolffd@0 121 % save
wolffd@0 122 save([hash(strcat(dirin{:}),'md5') '_summary'], 'out');
wolffd@0 123
wolffd@0 124 end
wolffd@0 125
wolffd@0 126
wolffd@0 127
wolffd@0 128 function boxplot(mean, std, train);
wolffd@0 129
wolffd@0 130 bar([train; mean]', 1.5);
wolffd@0 131 hold on;
wolffd@0 132 errorbar(1:size(mean,2), mean(1,:), std(1,:),'.');
wolffd@0 133 % plot(train,'rO');
wolffd@0 134 colormap(spring);
wolffd@0 135 axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]);
wolffd@0 136 end