wolffd@0
|
1 function [out]= do_test_rounds(trainfun, X, simdata, trainparams, fparams,...
|
wolffd@0
|
2 paramhash, paramhash_train, clips)
|
wolffd@0
|
3
|
wolffd@0
|
4 % ---
|
wolffd@0
|
5 % DEBUG: we mix up the training set
|
wolffd@0
|
6 % ---
|
wolffd@0
|
7 % simdata = mixup(simdata);
|
wolffd@0
|
8
|
wolffd@0
|
9 if isfield(simdata, 'clip_type');
|
wolffd@0
|
10 clip_type = simdata.clip_type;
|
wolffd@0
|
11 else
|
wolffd@0
|
12 clip_type = 'MTTClip';
|
wolffd@0
|
13 end
|
wolffd@0
|
14
|
wolffd@0
|
15 nTestSets = size(simdata.partBinTst, 2); % num cv bins
|
wolffd@0
|
16 ntrainsizes = size(simdata.partBinTrn, 2); % num increases of training
|
wolffd@0
|
17
|
wolffd@0
|
18 for m = 1:ntrainsizes
|
wolffd@0
|
19
|
wolffd@0
|
20 ok_train = zeros(2, nTestSets);
|
wolffd@0
|
21 ok_test = zeros(2, nTestSets);
|
wolffd@0
|
22 equal_test = zeros(1, nTestSets);
|
wolffd@0
|
23 ok_notin_train = zeros(2, nTestSets);
|
wolffd@0
|
24
|
wolffd@0
|
25 % A = cell(nTestSets,1);
|
wolffd@0
|
26 % dout = cell(nTestSets,1);
|
wolffd@0
|
27 % clips_train = cell(nTestSets,1);
|
wolffd@0
|
28 % clips_test = cell(nTestSets,1);
|
wolffd@0
|
29 % clips_notin_train = cell(nTestSets,1);
|
wolffd@0
|
30 % Y_notin_train = cell(nTestSets,1);
|
wolffd@0
|
31 % Ytrain = cell(nTestSets,1);
|
wolffd@0
|
32 % Ytest = cell(nTestSets,1);
|
wolffd@0
|
33 % parfor
|
wolffd@0
|
34 for k = 1:nTestSets
|
wolffd@0
|
35
|
wolffd@0
|
36
|
wolffd@0
|
37 % runlog mlr
|
wolffd@0
|
38 try
|
wolffd@0
|
39
|
wolffd@0
|
40 % ---
|
wolffd@0
|
41 % Get the training constraints and features for this round
|
wolffd@0
|
42 % ---
|
wolffd@0
|
43 % DEBUG: the similarity data in Ytrain and Ytest seems correct.
|
wolffd@0
|
44 [clips_train{k}, Xtrain, Ytrain{k}] ...
|
wolffd@0
|
45 = get_data_compact(clips, X, simdata.partBinTrn{k,m});
|
wolffd@0
|
46 Ytest{k}={};
|
wolffd@0
|
47
|
wolffd@0
|
48 % training step
|
wolffd@0
|
49 [A{k}, dout{k}] = feval(trainfun, Xtrain, Ytrain{k}, trainparams);
|
wolffd@0
|
50
|
wolffd@0
|
51 % ---
|
wolffd@0
|
52 % test step
|
wolffd@0
|
53 % TODO: the distmeasure object could be created by the wrapper!
|
wolffd@0
|
54 % ---
|
wolffd@0
|
55 if isfield(dout{k},'interpreter');
|
wolffd@0
|
56 interpreter = str2func(dout{k}.interpreter);
|
wolffd@0
|
57 else
|
wolffd@0
|
58 % only for backward compability
|
wolffd@0
|
59 % warning ('legacy implementation of dist measure');
|
wolffd@0
|
60 if isnumeric(A{k})
|
wolffd@0
|
61 % mahalanobis case
|
wolffd@0
|
62
|
wolffd@0
|
63 % special delta mahalanobis
|
wolffd@0
|
64 interpreter = str2func('DistMeasureMahal');
|
wolffd@0
|
65 else
|
wolffd@0
|
66 % neural network case: A{k} is a neural net object
|
wolffd@0
|
67 interpreter = str2func('DistMeasureGeneric');
|
wolffd@0
|
68 end
|
wolffd@0
|
69 end
|
wolffd@0
|
70
|
wolffd@0
|
71 if isfield(trainparams,'deltafun')
|
wolffd@0
|
72 % special delta
|
wolffd@0
|
73 diss = feval(interpreter,clips, A{k}, X, str2func(trainparams.deltafun), trainparams.deltafun_params);
|
wolffd@0
|
74 else
|
wolffd@0
|
75 % standard
|
wolffd@0
|
76 % ---
|
wolffd@0
|
77 % TODO: the default delta is different between
|
wolffd@0
|
78 % similarity measures. except for the mahalmeasure
|
wolffd@0
|
79 % this should be specified
|
wolffd@0
|
80 % ---
|
wolffd@0
|
81 diss = feval(interpreter, clips, A{k}, X);
|
wolffd@0
|
82 end
|
wolffd@0
|
83
|
wolffd@0
|
84 % test training data
|
wolffd@0
|
85 [ok_train(:,k)] = metric_fulfills_ranking...
|
wolffd@0
|
86 (diss, Ytrain{k}, feval(clip_type,clips_train{k}));
|
wolffd@0
|
87
|
wolffd@0
|
88 % get test data
|
wolffd@0
|
89 [clips_test{k}, Xtest, Ytest{k}] ...
|
wolffd@0
|
90 = get_data_compact(clips, X, simdata.partBinTst{k});
|
wolffd@0
|
91
|
wolffd@0
|
92 % diss = DistMeasureMahal(feval(clip_type,clips_test{k}), A{k}, Xtest);
|
wolffd@0
|
93 % test test data
|
wolffd@0
|
94 [ok_test(:,k), equal_test(k)] = metric_fulfills_ranking...
|
wolffd@0
|
95 (diss, Ytest{k}, feval(clip_type,clips_test{k}));
|
wolffd@0
|
96 cprint(3,'%2.2f %2.2f fold performance', ok_test(:,k));
|
wolffd@0
|
97
|
wolffd@0
|
98 % ---
|
wolffd@0
|
99 % extra diag for MLR
|
wolffd@0
|
100 % TODO: make this wrappeable
|
wolffd@0
|
101 % ---
|
wolffd@0
|
102 if isequal(trainfun, @mlr_wrapper)
|
wolffd@0
|
103 dout{k}.mprperf = mlr_test(A{k}, 0, Xtrain, Ytrain{k}(:,1:2), Xtest, Ytest{k}(:,1:2)) ;
|
wolffd@0
|
104 end
|
wolffd@0
|
105
|
wolffd@0
|
106 % ---
|
wolffd@0
|
107 % this gives data for the unused training set remainders
|
wolffd@0
|
108 % ---
|
wolffd@0
|
109 if isfield(simdata,'partBinNoTrn')
|
wolffd@0
|
110 if ~isempty(simdata.partBinNoTrn{k,m})
|
wolffd@0
|
111 [clips_notin_train{k}, X_notin_train, Y_notin_train{k}] ...
|
wolffd@0
|
112 = get_data_compact(clips, X, simdata.partBinNoTrn{k,m});
|
wolffd@0
|
113
|
wolffd@0
|
114 % test unused training data
|
wolffd@0
|
115 [ok_notin_train(:,k), equal_test(k)] = metric_fulfills_ranking...
|
wolffd@0
|
116 (diss, Y_notin_train{k}, feval(clip_type,clips_notin_train{k}));
|
wolffd@0
|
117
|
wolffd@0
|
118 % what to do if there is no data ?
|
wolffd@0
|
119 else
|
wolffd@0
|
120 ok_notin_train(:,k) = -1;
|
wolffd@0
|
121 end
|
wolffd@0
|
122 else
|
wolffd@0
|
123 ok_notin_train(:,k) = -1;
|
wolffd@0
|
124 end
|
wolffd@0
|
125
|
wolffd@0
|
126 catch err
|
wolffd@0
|
127
|
wolffd@0
|
128 % ---
|
wolffd@0
|
129 % in case training or test fails
|
wolffd@0
|
130 % ---
|
wolffd@0
|
131 print_error(err);
|
wolffd@0
|
132
|
wolffd@0
|
133 A{k} = [];
|
wolffd@0
|
134 dout{k} = -1;
|
wolffd@0
|
135
|
wolffd@0
|
136 ok_test(:,k) = -1;
|
wolffd@0
|
137 ok_train(:,k) = -1;
|
wolffd@0
|
138 ok_notin_train(:,k) = -1;
|
wolffd@0
|
139 equal_test(k) = -1;
|
wolffd@0
|
140
|
wolffd@0
|
141 % ---
|
wolffd@0
|
142 % save feature, system and data configuration
|
wolffd@0
|
143 % and indicate failure
|
wolffd@0
|
144 % ---
|
wolffd@0
|
145 xml_save(sprintf('runlog_%s.%s_trainparam.xml',...
|
wolffd@0
|
146 paramhash, paramhash_train), trainparams);
|
wolffd@0
|
147 xml_save(sprintf('runlog_%s.%s_err.xml',...
|
wolffd@0
|
148 paramhash, paramhash_train), print_error(err));
|
wolffd@0
|
149 end
|
wolffd@0
|
150 end
|
wolffd@0
|
151
|
wolffd@0
|
152 if ~(ntrainsizes == 1)
|
wolffd@0
|
153
|
wolffd@0
|
154 % save elaborate testing data
|
wolffd@0
|
155 size_sum = 0;
|
wolffd@0
|
156 for i = 1:nTestSets
|
wolffd@0
|
157 size_sum = size_sum + size(simdata.partBinTrn{i,m}) / size(simdata.partBinTrn{i,end});
|
wolffd@0
|
158 end
|
wolffd@0
|
159 size_sum = size_sum / nTestSets;
|
wolffd@0
|
160
|
wolffd@0
|
161 out.inctrain.trainfrac(:, m) = size_sum;
|
wolffd@0
|
162 out.inctrain.dataPartition(:, m) = 0;
|
wolffd@0
|
163
|
wolffd@0
|
164 % ---
|
wolffd@0
|
165 % NOTE: the max value is important for debugging,
|
wolffd@0
|
166 % especially when the maximal training success is reached
|
wolffd@0
|
167 % in the middle of the data set
|
wolffd@0
|
168 % ---
|
wolffd@0
|
169 % out.inctrain.max_ok_test(:, m) = max(ok_test, 2);
|
wolffd@0
|
170 out.inctrain.mean_ok_test(:, m) = mean(ok_test(:, ok_test(1,:) >=0), 2);
|
wolffd@0
|
171 out.inctrain.var_ok_test(:, m) = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
|
wolffd@0
|
172 out.inctrain.equal_test(m) = median(equal_test);
|
wolffd@0
|
173
|
wolffd@0
|
174 out.inctrain.mean_ok_train(:, m) = mean(ok_train(:, ok_train(1,:) >=0), 2);
|
wolffd@0
|
175 out.inctrain.var_ok_train(:, m) = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
|
wolffd@0
|
176
|
wolffd@0
|
177 % ---
|
wolffd@0
|
178 % TODO: DEBUG: this does not work correctly
|
wolffd@0
|
179 % maybe thats also true for the above?
|
wolffd@0
|
180 % ---
|
wolffd@0
|
181 out.inctrain.mean_ok_notin_train(:, m) = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
|
wolffd@0
|
182 out.inctrain.var_ok_notin_train(:, m) = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);
|
wolffd@0
|
183
|
wolffd@0
|
184 diag.inctrain(m).ok_train = ok_train;
|
wolffd@0
|
185 diag.inctrain(m).ok_test = ok_test;
|
wolffd@0
|
186 diag.inctrain(m).ok_notin_train = ok_notin_train;
|
wolffd@0
|
187 diag.inctrain(m).equal_test = equal_test;
|
wolffd@0
|
188 end
|
wolffd@0
|
189
|
wolffd@0
|
190 % ---
|
wolffd@0
|
191 % save traditional information for full training set
|
wolffd@0
|
192 % ---
|
wolffd@0
|
193 if size(simdata.partBinTrn{1,m}) == size(simdata.partBinTrn{1,end});
|
wolffd@0
|
194
|
wolffd@0
|
195 % out.max_ok_test = max(ok_test, 2);
|
wolffd@0
|
196 out.mean_ok_test = mean(ok_test(:, ok_test(1,:) >=0), 2);
|
wolffd@0
|
197 out.var_ok_test = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
|
wolffd@0
|
198 out.equal_test = median(equal_test);
|
wolffd@0
|
199
|
wolffd@0
|
200 out.mean_ok_train = mean(ok_train(:, ok_train(1,:) >=0), 2);
|
wolffd@0
|
201 out.var_ok_train = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
|
wolffd@0
|
202
|
wolffd@0
|
203 % ---
|
wolffd@0
|
204 % TODO: DEBUG: this does not work correctly
|
wolffd@0
|
205 % ---
|
wolffd@0
|
206 out.mean_ok_notin_train = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
|
wolffd@0
|
207 out.var_ok_notin_train = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);
|
wolffd@0
|
208
|
wolffd@0
|
209 % ---
|
wolffd@0
|
210 % get winning measure
|
wolffd@0
|
211 % we use the weighted winning measure if possible
|
wolffd@0
|
212 % ---
|
wolffd@0
|
213 if max(ok_test(2,:)) > 0
|
wolffd@0
|
214 [~, best] = max(ok_test(2,:));
|
wolffd@0
|
215 else
|
wolffd@0
|
216 [~, best] = max(ok_test(1,:));
|
wolffd@0
|
217 end
|
wolffd@0
|
218
|
wolffd@0
|
219 diag.A = A;
|
wolffd@0
|
220 diag.diag = dout;
|
wolffd@0
|
221
|
wolffd@0
|
222 diag.ok_test = ok_test;
|
wolffd@0
|
223 diag.equal_test = equal_test;
|
wolffd@0
|
224 diag.ok_train = ok_train;
|
wolffd@0
|
225 diag.ok_notin_train = ok_notin_train;
|
wolffd@0
|
226
|
wolffd@0
|
227 % save some metric matrices
|
wolffd@0
|
228 out.best_A = A{best};
|
wolffd@0
|
229 out.best_diag = dout{best};
|
wolffd@0
|
230 out.best_idx = best;
|
wolffd@0
|
231
|
wolffd@0
|
232 end
|
wolffd@0
|
233 end
|
wolffd@0
|
234
|
wolffd@0
|
235 % save parameters
|
wolffd@0
|
236 out.camirrev = camirversion();
|
wolffd@0
|
237 out.fparams = fparams;
|
wolffd@0
|
238 out.trainfun = trainfun;
|
wolffd@0
|
239 out.trainparams = trainparams;
|
wolffd@0
|
240 out.clip_ids = clips.id();
|
wolffd@0
|
241 out.dataPartition = [];
|
wolffd@0
|
242 out.Y = size(simdata);
|
wolffd@0
|
243 % ---
|
wolffd@0
|
244 % NOTE: this takes A LOT OF DISC SPACE
|
wolffd@0
|
245 % ---
|
wolffd@0
|
246 % out.Ytrain = Ytrain{end};
|
wolffd@0
|
247 % out.Ytest = Ytest{end};
|
wolffd@0
|
248
|
wolffd@0
|
249 % ---
|
wolffd@0
|
250 % save the diagostics data to disk
|
wolffd@0
|
251 % ---
|
wolffd@0
|
252 save(sprintf('runlog_%s.%s_results.mat',...
|
wolffd@0
|
253 paramhash, paramhash_train),...
|
wolffd@0
|
254 'out', 'diag');
|
wolffd@0
|
255 end |