Mercurial > hg > camir-aes2014

annotate core/magnatagatune/tests_evals/test_generic_display_results.m @ 0:e9a9cd732c1e tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first hg version after svn
author wolffd
date Tue, 10 Feb 2015 15:05:51 +0000
parents
children
rev   line source
wolffd@0 1 function [out, stats, features, individual] = test_generic_display_results(file, get_features, show)
wolffd@0 2 % [out, stats] = test_generic_display_results([file], get_features)
wolffd@0 3 %
wolffd@0 4 % [out, stats, features, individual] = test_generic_display_results([file], get_features)
wolffd@0 5 %
wolffd@0 6 % displays the finalresults mat file and enables
wolffd@0 7 % further analysis and duiagnostics of the individual runs
wolffd@0 8
wolffd@0 9 features = [];
wolffd@0 10
wolffd@0 11 if nargin < 3
wolffd@0 12 show = 1;
wolffd@0 13 end
wolffd@0 14 if nargin < 2
wolffd@0 15 get_features = 1;
wolffd@0 16 end
wolffd@0 17
wolffd@0 18 global comparison;
wolffd@0 19 global comparison_ids;
wolffd@0 20
wolffd@0 21 if nargin < 1 || isempty(file) || isnumeric(file)
wolffd@0 22 u = dir();
wolffd@0 23 u = {u.name};
wolffd@0 24 [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1);
wolffd@0 25
wolffd@0 26 if numel(idx) < 1
wolffd@0 27 error 'This directory contains no valid test data';
wolffd@0 28 end
wolffd@0 29
wolffd@0 30 if exist('file','var') && isnumeric(file)
wolffd@0 31 file = u{idx(file)};
wolffd@0 32 else
wolffd@0 33 if numel(idx) > 1
wolffd@0 34 file = u{idx(ask_dataset())};
wolffd@0 35 else
wolffd@0 36 file = u{idx(1)};
wolffd@0 37 end
wolffd@0 38 end
wolffd@0 39 end
wolffd@0 40
wolffd@0 41
wolffd@0 42 % ---
wolffd@0 43 % LOAD THE RESULT DATA
wolffd@0 44 % We have:
wolffd@0 45 % Y
wolffd@0 46 % out.fparams
wolffd@0 47 % trainparams
wolffd@0 48 % dataPartition
wolffd@0 49 % mean_ok_test
wolffd@0 50 % var_ok_test
wolffd@0 51 % mean_ok_train
wolffd@0 52 % ---
wolffd@0 53 load(file);
wolffd@0 54
wolffd@0 55 % compability
wolffd@0 56 if isfield(out, 'mlrparams')
wolffd@0 57 for i = 1:numel(out)
wolffd@0 58 out(i).trainparams = out(i).mlrparams;
wolffd@0 59 end
wolffd@0 60 end
wolffd@0 61
wolffd@0 62
wolffd@0 63 % ---
wolffd@0 64 % % get statistics for feature parameters
wolffd@0 65 % Visualise the accuracy and variance
wolffd@0 66 % ---
wolffd@0 67 if isfield(out, 'inctrain') && show
wolffd@0 68 for i = 1:numel(out)
wolffd@0 69
wolffd@0 70 figure;
wolffd@0 71 boxplot([out(i).inctrain.mean_ok_test], sqrt([out(i).inctrain.var_ok_test]), [out(i).inctrain.mean_ok_train]);
wolffd@0 72 set(gca,'XTick',1:numel(out(i).inctrain.trainfrac), ...
wolffd@0 73 'XTickLabel', out(i).inctrain.trainfrac* 100);
wolffd@0 74
wolffd@0 75 xlabel ('fraction of training data');
wolffd@0 76 title (sprintf('increasing training size test, config %d',i));
wolffd@0 77 legend('train', 'train weighted', 'test', 'test weighted');
wolffd@0 78
wolffd@0 79 end
wolffd@0 80 end
wolffd@0 81
wolffd@0 82
wolffd@0 83 if numel([out.mean_ok_test]) > 1 && show
wolffd@0 84
wolffd@0 85 % plot means % plot std = sqrt(var) % plot training results
wolffd@0 86 figure;
wolffd@0 87 boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]);
wolffd@0 88 title (sprintf('Performance for all configs'));
wolffd@0 89 end
wolffd@0 90
wolffd@0 91
wolffd@0 92 % ---
wolffd@0 93 % write max. test success
wolffd@0 94 % ---
wolffd@0 95 mean_ok_test = [out.mean_ok_test];
wolffd@0 96 [val, idx] = max(mean_ok_test(1,:));
wolffd@0 97 if show
wolffd@0 98 fprintf(' --- Maximal test set success: nr. %d, %3.2f percent. --- \n', idx, val * 100)
wolffd@0 99 end
wolffd@0 100
wolffd@0 101 % ---
wolffd@0 102 % display parameter statistics
wolffd@0 103 % ---
wolffd@0 104 stats = test_generic_display_param_influence(out, show);
wolffd@0 105
wolffd@0 106
wolffd@0 107 if nargout < 3
wolffd@0 108 return;
wolffd@0 109 end
wolffd@0 110 % ---
wolffd@0 111 % display statistics and get features
wolffd@0 112 % for run with best test success
wolffd@0 113 % ---
wolffd@0 114 [resfile, featfile] = get_res_filename(out, idx);
wolffd@0 115
wolffd@0 116 % ---
wolffd@0 117 % import features:
wolffd@0 118 % 1. reset databse
wolffd@0 119 % 2. import features
wolffd@0 120 % 3. assign to clip ids as in ranking
wolffd@0 121 % ---
wolffd@0 122 if get_features
wolffd@0 123 type = MTTAudioFeatureDBgen.import_type(featfile);
wolffd@0 124 db_name = MTTAudioFeatureDBgen.db_name(type);
wolffd@0 125 eval(sprintf('global %s', db_name));
wolffd@0 126 eval(sprintf('%s.reset();', db_name));
wolffd@0 127 eval(sprintf('features = %s.import(featfile);', db_name));
wolffd@0 128
wolffd@0 129 if isfield(out,'clip_ids')
wolffd@0 130 clips = MTTClip(out(1).clip_ids);
wolffd@0 131 features = clips.features(type);
wolffd@0 132 end
wolffd@0 133 end
wolffd@0 134
wolffd@0 135 % ---
wolffd@0 136 % Display Metric Stats
wolffd@0 137 % tmp = test_mlr_display_metric_stats(individual.out, individual.diag, features);
wolffd@0 138 % ---
wolffd@0 139
wolffd@0 140 if nargout < 4
wolffd@0 141 return;
wolffd@0 142 end
wolffd@0 143 individual = load(resfile);
wolffd@0 144 for i = 1:numel(out)
wolffd@0 145
wolffd@0 146 [resfile, featfile] = get_res_filename(out, i);
wolffd@0 147
wolffd@0 148 if get_features
wolffd@0 149 % reset db and load testing features
wolffd@0 150 eval(sprintf('global %s', db_name));
wolffd@0 151 eval(sprintf('%s.reset();', db_name));
wolffd@0 152 eval(sprintf('%s.import(featfile);', db_name));
wolffd@0 153 end
wolffd@0 154
wolffd@0 155 % load individual results
wolffd@0 156 if i == 1;
wolffd@0 157
wolffd@0 158 individual = load(resfile);
wolffd@0 159 else
wolffd@0 160
wolffd@0 161 individual(i) = load(resfile);
wolffd@0 162 end
wolffd@0 163 end
wolffd@0 164 end
wolffd@0 165
wolffd@0 166 function out = ask_dataset()
wolffd@0 167 % ---
wolffd@0 168 % displays the parameters of the datasets,
wolffd@0 169 % and asks for the right one to display
wolffd@0 170 % ---
wolffd@0 171 clc;
wolffd@0 172 u = dir();
wolffd@0 173 u = {u.name};
wolffd@0 174 [idx, strpos] = substrcellfind(u, '_params.mat', 1);
wolffd@0 175
wolffd@0 176 for i = 1:numel(idx)
wolffd@0 177 file = u{idx(i)};
wolffd@0 178 fprintf('------------ Dataset nr. %d --------------\n',i);
wolffd@0 179 fprintf('Filename: %s\n',file);
wolffd@0 180 type(file);
wolffd@0 181 end
wolffd@0 182
wolffd@0 183 out = (input('Please choose the dataset number: '));
wolffd@0 184 end
wolffd@0 185
wolffd@0 186
wolffd@0 187 function [resfile, featfile] = get_res_filename(out, i)
wolffd@0 188 % get filename given test results and index
wolffd@0 189
wolffd@0 190 paramhash = hash(xml_format(out(i).fparams),'MD5');
wolffd@0 191
wolffd@0 192 paramhash_mlr = hash(xml_format(out(i).trainparams),'MD5');
wolffd@0 193
wolffd@0 194 featfile = sprintf('runlog_%s_feat.mat', paramhash);
wolffd@0 195
wolffd@0 196 resfile = sprintf('runlog_%s.%s_results.mat',...
wolffd@0 197 paramhash, paramhash_mlr);
wolffd@0 198 end
wolffd@0 199
wolffd@0 200
wolffd@0 201 function boxplot(mean, std, train);
wolffd@0 202
wolffd@0 203 bar([train; mean]', 1.5);
wolffd@0 204 hold on;
wolffd@0 205 errorbar(1:size(mean,2), mean(1,:), std(1,:),'.');
wolffd@0 206 % plot(train,'rO');
wolffd@0 207 colormap(spring);
wolffd@0 208 axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]);
wolffd@0 209 end