Mercurial > hg > camir-aes2014

annotate core/magnatagatune/tests_evals/do_test_rounds.m @ 0:e9a9cd732c1e tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first hg version after svn
author wolffd
date Tue, 10 Feb 2015 15:05:51 +0000
parents
children
rev   line source
wolffd@0 1 function [out]= do_test_rounds(trainfun, X, simdata, trainparams, fparams,...
wolffd@0 2 paramhash, paramhash_train, clips)
wolffd@0 3
wolffd@0 4 % ---
wolffd@0 5 % DEBUG: we mix up the training set
wolffd@0 6 % ---
wolffd@0 7 % simdata = mixup(simdata);
wolffd@0 8
wolffd@0 9 if isfield(simdata, 'clip_type');
wolffd@0 10 clip_type = simdata.clip_type;
wolffd@0 11 else
wolffd@0 12 clip_type = 'MTTClip';
wolffd@0 13 end
wolffd@0 14
wolffd@0 15 nTestSets = size(simdata.partBinTst, 2); % num cv bins
wolffd@0 16 ntrainsizes = size(simdata.partBinTrn, 2); % num increases of training
wolffd@0 17
wolffd@0 18 for m = 1:ntrainsizes
wolffd@0 19
wolffd@0 20 ok_train = zeros(2, nTestSets);
wolffd@0 21 ok_test = zeros(2, nTestSets);
wolffd@0 22 equal_test = zeros(1, nTestSets);
wolffd@0 23 ok_notin_train = zeros(2, nTestSets);
wolffd@0 24
wolffd@0 25 % A = cell(nTestSets,1);
wolffd@0 26 % dout = cell(nTestSets,1);
wolffd@0 27 % clips_train = cell(nTestSets,1);
wolffd@0 28 % clips_test = cell(nTestSets,1);
wolffd@0 29 % clips_notin_train = cell(nTestSets,1);
wolffd@0 30 % Y_notin_train = cell(nTestSets,1);
wolffd@0 31 % Ytrain = cell(nTestSets,1);
wolffd@0 32 % Ytest = cell(nTestSets,1);
wolffd@0 33 % parfor
wolffd@0 34 for k = 1:nTestSets
wolffd@0 35
wolffd@0 36
wolffd@0 37 % runlog mlr
wolffd@0 38 try
wolffd@0 39
wolffd@0 40 % ---
wolffd@0 41 % Get the training constraints and features for this round
wolffd@0 42 % ---
wolffd@0 43 % DEBUG: the similarity data in Ytrain and Ytest seems correct.
wolffd@0 44 [clips_train{k}, Xtrain, Ytrain{k}] ...
wolffd@0 45 = get_data_compact(clips, X, simdata.partBinTrn{k,m});
wolffd@0 46 Ytest{k}={};
wolffd@0 47
wolffd@0 48 % training step
wolffd@0 49 [A{k}, dout{k}] = feval(trainfun, Xtrain, Ytrain{k}, trainparams);
wolffd@0 50
wolffd@0 51 % ---
wolffd@0 52 % test step
wolffd@0 53 % TODO: the distmeasure object could be created by the wrapper!
wolffd@0 54 % ---
wolffd@0 55 if isfield(dout{k},'interpreter');
wolffd@0 56 interpreter = str2func(dout{k}.interpreter);
wolffd@0 57 else
wolffd@0 58 % only for backward compability
wolffd@0 59 % warning ('legacy implementation of dist measure');
wolffd@0 60 if isnumeric(A{k})
wolffd@0 61 % mahalanobis case
wolffd@0 62
wolffd@0 63 % special delta mahalanobis
wolffd@0 64 interpreter = str2func('DistMeasureMahal');
wolffd@0 65 else
wolffd@0 66 % neural network case: A{k} is a neural net object
wolffd@0 67 interpreter = str2func('DistMeasureGeneric');
wolffd@0 68 end
wolffd@0 69 end
wolffd@0 70
wolffd@0 71 if isfield(trainparams,'deltafun')
wolffd@0 72 % special delta
wolffd@0 73 diss = feval(interpreter,clips, A{k}, X, str2func(trainparams.deltafun), trainparams.deltafun_params);
wolffd@0 74 else
wolffd@0 75 % standard
wolffd@0 76 % ---
wolffd@0 77 % TODO: the default delta is different between
wolffd@0 78 % similarity measures. except for the mahalmeasure
wolffd@0 79 % this should be specified
wolffd@0 80 % ---
wolffd@0 81 diss = feval(interpreter, clips, A{k}, X);
wolffd@0 82 end
wolffd@0 83
wolffd@0 84 % test training data
wolffd@0 85 [ok_train(:,k)] = metric_fulfills_ranking...
wolffd@0 86 (diss, Ytrain{k}, feval(clip_type,clips_train{k}));
wolffd@0 87
wolffd@0 88 % get test data
wolffd@0 89 [clips_test{k}, Xtest, Ytest{k}] ...
wolffd@0 90 = get_data_compact(clips, X, simdata.partBinTst{k});
wolffd@0 91
wolffd@0 92 % diss = DistMeasureMahal(feval(clip_type,clips_test{k}), A{k}, Xtest);
wolffd@0 93 % test test data
wolffd@0 94 [ok_test(:,k), equal_test(k)] = metric_fulfills_ranking...
wolffd@0 95 (diss, Ytest{k}, feval(clip_type,clips_test{k}));
wolffd@0 96 cprint(3,'%2.2f %2.2f fold performance', ok_test(:,k));
wolffd@0 97
wolffd@0 98 % ---
wolffd@0 99 % extra diag for MLR
wolffd@0 100 % TODO: make this wrappeable
wolffd@0 101 % ---
wolffd@0 102 if isequal(trainfun, @mlr_wrapper)
wolffd@0 103 dout{k}.mprperf = mlr_test(A{k}, 0, Xtrain, Ytrain{k}(:,1:2), Xtest, Ytest{k}(:,1:2)) ;
wolffd@0 104 end
wolffd@0 105
wolffd@0 106 % ---
wolffd@0 107 % this gives data for the unused training set remainders
wolffd@0 108 % ---
wolffd@0 109 if isfield(simdata,'partBinNoTrn')
wolffd@0 110 if ~isempty(simdata.partBinNoTrn{k,m})
wolffd@0 111 [clips_notin_train{k}, X_notin_train, Y_notin_train{k}] ...
wolffd@0 112 = get_data_compact(clips, X, simdata.partBinNoTrn{k,m});
wolffd@0 113
wolffd@0 114 % test unused training data
wolffd@0 115 [ok_notin_train(:,k), equal_test(k)] = metric_fulfills_ranking...
wolffd@0 116 (diss, Y_notin_train{k}, feval(clip_type,clips_notin_train{k}));
wolffd@0 117
wolffd@0 118 % what to do if there is no data ?
wolffd@0 119 else
wolffd@0 120 ok_notin_train(:,k) = -1;
wolffd@0 121 end
wolffd@0 122 else
wolffd@0 123 ok_notin_train(:,k) = -1;
wolffd@0 124 end
wolffd@0 125
wolffd@0 126 catch err
wolffd@0 127
wolffd@0 128 % ---
wolffd@0 129 % in case training or test fails
wolffd@0 130 % ---
wolffd@0 131 print_error(err);
wolffd@0 132
wolffd@0 133 A{k} = [];
wolffd@0 134 dout{k} = -1;
wolffd@0 135
wolffd@0 136 ok_test(:,k) = -1;
wolffd@0 137 ok_train(:,k) = -1;
wolffd@0 138 ok_notin_train(:,k) = -1;
wolffd@0 139 equal_test(k) = -1;
wolffd@0 140
wolffd@0 141 % ---
wolffd@0 142 % save feature, system and data configuration
wolffd@0 143 % and indicate failure
wolffd@0 144 % ---
wolffd@0 145 xml_save(sprintf('runlog_%s.%s_trainparam.xml',...
wolffd@0 146 paramhash, paramhash_train), trainparams);
wolffd@0 147 xml_save(sprintf('runlog_%s.%s_err.xml',...
wolffd@0 148 paramhash, paramhash_train), print_error(err));
wolffd@0 149 end
wolffd@0 150 end
wolffd@0 151
wolffd@0 152 if ~(ntrainsizes == 1)
wolffd@0 153
wolffd@0 154 % save elaborate testing data
wolffd@0 155 size_sum = 0;
wolffd@0 156 for i = 1:nTestSets
wolffd@0 157 size_sum = size_sum + size(simdata.partBinTrn{i,m}) / size(simdata.partBinTrn{i,end});
wolffd@0 158 end
wolffd@0 159 size_sum = size_sum / nTestSets;
wolffd@0 160
wolffd@0 161 out.inctrain.trainfrac(:, m) = size_sum;
wolffd@0 162 out.inctrain.dataPartition(:, m) = 0;
wolffd@0 163
wolffd@0 164 % ---
wolffd@0 165 % NOTE: the max value is important for debugging,
wolffd@0 166 % especially when the maximal training success is reached
wolffd@0 167 % in the middle of the data set
wolffd@0 168 % ---
wolffd@0 169 % out.inctrain.max_ok_test(:, m) = max(ok_test, 2);
wolffd@0 170 out.inctrain.mean_ok_test(:, m) = mean(ok_test(:, ok_test(1,:) >=0), 2);
wolffd@0 171 out.inctrain.var_ok_test(:, m) = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
wolffd@0 172 out.inctrain.equal_test(m) = median(equal_test);
wolffd@0 173
wolffd@0 174 out.inctrain.mean_ok_train(:, m) = mean(ok_train(:, ok_train(1,:) >=0), 2);
wolffd@0 175 out.inctrain.var_ok_train(:, m) = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
wolffd@0 176
wolffd@0 177 % ---
wolffd@0 178 % TODO: DEBUG: this does not work correctly
wolffd@0 179 % maybe thats also true for the above?
wolffd@0 180 % ---
wolffd@0 181 out.inctrain.mean_ok_notin_train(:, m) = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
wolffd@0 182 out.inctrain.var_ok_notin_train(:, m) = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);
wolffd@0 183
wolffd@0 184 diag.inctrain(m).ok_train = ok_train;
wolffd@0 185 diag.inctrain(m).ok_test = ok_test;
wolffd@0 186 diag.inctrain(m).ok_notin_train = ok_notin_train;
wolffd@0 187 diag.inctrain(m).equal_test = equal_test;
wolffd@0 188 end
wolffd@0 189
wolffd@0 190 % ---
wolffd@0 191 % save traditional information for full training set
wolffd@0 192 % ---
wolffd@0 193 if size(simdata.partBinTrn{1,m}) == size(simdata.partBinTrn{1,end});
wolffd@0 194
wolffd@0 195 % out.max_ok_test = max(ok_test, 2);
wolffd@0 196 out.mean_ok_test = mean(ok_test(:, ok_test(1,:) >=0), 2);
wolffd@0 197 out.var_ok_test = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
wolffd@0 198 out.equal_test = median(equal_test);
wolffd@0 199
wolffd@0 200 out.mean_ok_train = mean(ok_train(:, ok_train(1,:) >=0), 2);
wolffd@0 201 out.var_ok_train = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
wolffd@0 202
wolffd@0 203 % ---
wolffd@0 204 % TODO: DEBUG: this does not work correctly
wolffd@0 205 % ---
wolffd@0 206 out.mean_ok_notin_train = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
wolffd@0 207 out.var_ok_notin_train = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);
wolffd@0 208
wolffd@0 209 % ---
wolffd@0 210 % get winning measure
wolffd@0 211 % we use the weighted winning measure if possible
wolffd@0 212 % ---
wolffd@0 213 if max(ok_test(2,:)) > 0
wolffd@0 214 [~, best] = max(ok_test(2,:));
wolffd@0 215 else
wolffd@0 216 [~, best] = max(ok_test(1,:));
wolffd@0 217 end
wolffd@0 218
wolffd@0 219 diag.A = A;
wolffd@0 220 diag.diag = dout;
wolffd@0 221
wolffd@0 222 diag.ok_test = ok_test;
wolffd@0 223 diag.equal_test = equal_test;
wolffd@0 224 diag.ok_train = ok_train;
wolffd@0 225 diag.ok_notin_train = ok_notin_train;
wolffd@0 226
wolffd@0 227 % save some metric matrices
wolffd@0 228 out.best_A = A{best};
wolffd@0 229 out.best_diag = dout{best};
wolffd@0 230 out.best_idx = best;
wolffd@0 231
wolffd@0 232 end
wolffd@0 233 end
wolffd@0 234
wolffd@0 235 % save parameters
wolffd@0 236 out.camirrev = camirversion();
wolffd@0 237 out.fparams = fparams;
wolffd@0 238 out.trainfun = trainfun;
wolffd@0 239 out.trainparams = trainparams;
wolffd@0 240 out.clip_ids = clips.id();
wolffd@0 241 out.dataPartition = [];
wolffd@0 242 out.Y = size(simdata);
wolffd@0 243 % ---
wolffd@0 244 % NOTE: this takes A LOT OF DISC SPACE
wolffd@0 245 % ---
wolffd@0 246 % out.Ytrain = Ytrain{end};
wolffd@0 247 % out.Ytest = Ytest{end};
wolffd@0 248
wolffd@0 249 % ---
wolffd@0 250 % save the diagostics data to disk
wolffd@0 251 % ---
wolffd@0 252 save(sprintf('runlog_%s.%s_results.mat',...
wolffd@0 253 paramhash, paramhash_train),...
wolffd@0 254 'out', 'diag');
wolffd@0 255 end