wolffd@0: function [out]= do_test_rounds(trainfun, X, simdata, trainparams, fparams,...
wolffd@0:             paramhash, paramhash_train, clips)
wolffd@0: 
wolffd@0: % ---
wolffd@0: % DEBUG: we mix up the training set
wolffd@0: % ---
wolffd@0: % simdata = mixup(simdata);
wolffd@0: 
wolffd@0: if isfield(simdata, 'clip_type');
wolffd@0:     clip_type = simdata.clip_type;
wolffd@0: else
wolffd@0:     clip_type = 'MTTClip';
wolffd@0: end
wolffd@0: 
wolffd@0: nTestSets = size(simdata.partBinTst, 2); % num cv bins
wolffd@0: ntrainsizes = size(simdata.partBinTrn, 2); % num increases of training
wolffd@0: 
wolffd@0: for m = 1:ntrainsizes
wolffd@0: 
wolffd@0:     ok_train = zeros(2, nTestSets);
wolffd@0:     ok_test = zeros(2, nTestSets);
wolffd@0:     equal_test = zeros(1, nTestSets);
wolffd@0:     ok_notin_train = zeros(2, nTestSets);
wolffd@0:     
wolffd@0: %     A = cell(nTestSets,1);
wolffd@0: %     dout = cell(nTestSets,1);
wolffd@0: %     clips_train = cell(nTestSets,1);
wolffd@0: %     clips_test = cell(nTestSets,1);
wolffd@0: %     clips_notin_train = cell(nTestSets,1);
wolffd@0: %     Y_notin_train = cell(nTestSets,1);
wolffd@0: %     Ytrain = cell(nTestSets,1);
wolffd@0: %     Ytest = cell(nTestSets,1);
wolffd@0:     % parfor
wolffd@0:     for k = 1:nTestSets
wolffd@0: 
wolffd@0: 
wolffd@0:         % runlog mlr
wolffd@0:         try
wolffd@0:             
wolffd@0:             % ---
wolffd@0:             % Get the training constraints and features for this round
wolffd@0:             % ---
wolffd@0:             % DEBUG: the similarity data in Ytrain and Ytest seems correct.
wolffd@0:             [clips_train{k}, Xtrain, Ytrain{k}] ...
wolffd@0:                 = get_data_compact(clips, X, simdata.partBinTrn{k,m});
wolffd@0:             Ytest{k}={};
wolffd@0:             
wolffd@0:             % training step
wolffd@0:             [A{k}, dout{k}] = feval(trainfun, Xtrain, Ytrain{k}, trainparams);
wolffd@0: 
wolffd@0:             % ---
wolffd@0:             % test step
wolffd@0:             % TODO: the distmeasure object could be created by the wrapper! 
wolffd@0:             % ---
wolffd@0:             if isfield(dout{k},'interpreter');
wolffd@0:                 interpreter = str2func(dout{k}.interpreter);
wolffd@0:             else
wolffd@0:                 % only for backward compability
wolffd@0:                 % warning ('legacy implementation of dist measure');
wolffd@0:                  if isnumeric(A{k}) 
wolffd@0:                     % mahalanobis case
wolffd@0: 
wolffd@0:                     % special delta mahalanobis
wolffd@0:                     interpreter = str2func('DistMeasureMahal');
wolffd@0:                 else 
wolffd@0:                     % neural network case: A{k} is a neural net object
wolffd@0:                     interpreter = str2func('DistMeasureGeneric');
wolffd@0:                 end
wolffd@0:             end
wolffd@0:             
wolffd@0:             if isfield(trainparams,'deltafun')
wolffd@0:                 % special delta 
wolffd@0:                 diss = feval(interpreter,clips, A{k}, X, str2func(trainparams.deltafun), trainparams.deltafun_params);
wolffd@0:             else
wolffd@0:                 % standard 
wolffd@0:                 % ---
wolffd@0:                 % TODO: the default delta is different between 
wolffd@0:                 % similarity measures. except for the mahalmeasure
wolffd@0:                 % this should be specified
wolffd@0:                 % ---
wolffd@0:                 diss = feval(interpreter, clips, A{k}, X);
wolffd@0:             end
wolffd@0:             
wolffd@0:             % test training data 
wolffd@0:              [ok_train(:,k)] = metric_fulfills_ranking...
wolffd@0:                 (diss, Ytrain{k}, feval(clip_type,clips_train{k}));
wolffd@0:             
wolffd@0:             % get test data  
wolffd@0:             [clips_test{k}, Xtest, Ytest{k}] ...
wolffd@0:                 = get_data_compact(clips, X, simdata.partBinTst{k});
wolffd@0:             
wolffd@0:            % diss = DistMeasureMahal(feval(clip_type,clips_test{k}), A{k}, Xtest);
wolffd@0:             % test test data
wolffd@0:             [ok_test(:,k), equal_test(k)] = metric_fulfills_ranking...
wolffd@0:                 (diss, Ytest{k}, feval(clip_type,clips_test{k}));
wolffd@0:             cprint(3,'%2.2f %2.2f fold performance', ok_test(:,k));
wolffd@0:             
wolffd@0:             % ---
wolffd@0:             % extra diag for MLR
wolffd@0:             % TODO: make this wrappeable
wolffd@0:             % ---
wolffd@0:             if isequal(trainfun, @mlr_wrapper)
wolffd@0:                 dout{k}.mprperf =  mlr_test(A{k}, 0, Xtrain, Ytrain{k}(:,1:2), Xtest, Ytest{k}(:,1:2)) ;
wolffd@0:             end
wolffd@0:             
wolffd@0:             % --- 
wolffd@0:             % this gives data for the unused training set remainders
wolffd@0:             % ---
wolffd@0:             if isfield(simdata,'partBinNoTrn') 
wolffd@0:                 if ~isempty(simdata.partBinNoTrn{k,m})
wolffd@0:                 [clips_notin_train{k}, X_notin_train, Y_notin_train{k}] ...
wolffd@0:                     = get_data_compact(clips, X, simdata.partBinNoTrn{k,m});
wolffd@0:            
wolffd@0:                 % test unused training data
wolffd@0:                 [ok_notin_train(:,k), equal_test(k)] = metric_fulfills_ranking...
wolffd@0:                     (diss, Y_notin_train{k}, feval(clip_type,clips_notin_train{k}));
wolffd@0:                 
wolffd@0:                 % what to do if there is no data ? 
wolffd@0:                 else
wolffd@0:                     ok_notin_train(:,k) = -1;
wolffd@0:                 end
wolffd@0:             else
wolffd@0:                 ok_notin_train(:,k) = -1;
wolffd@0:             end
wolffd@0: 
wolffd@0:         catch err
wolffd@0:             
wolffd@0:             % ---
wolffd@0:             % in case training or test fails
wolffd@0:             % ---
wolffd@0:             print_error(err);
wolffd@0:             
wolffd@0:             A{k} = [];
wolffd@0:             dout{k} = -1;
wolffd@0: 
wolffd@0:             ok_test(:,k) = -1;
wolffd@0:             ok_train(:,k) = -1;
wolffd@0:             ok_notin_train(:,k) = -1;
wolffd@0:             equal_test(k) = -1;
wolffd@0:             
wolffd@0:             % ---
wolffd@0:             % save feature, system and data configuration
wolffd@0:             % and indicate failure
wolffd@0:             % ---
wolffd@0:             xml_save(sprintf('runlog_%s.%s_trainparam.xml',...
wolffd@0:                 paramhash, paramhash_train), trainparams);
wolffd@0:             xml_save(sprintf('runlog_%s.%s_err.xml',...
wolffd@0:                 paramhash, paramhash_train), print_error(err));
wolffd@0:         end
wolffd@0:     end
wolffd@0: 
wolffd@0:     if ~(ntrainsizes == 1)
wolffd@0:         
wolffd@0:         % save elaborate testing data
wolffd@0:         size_sum = 0;
wolffd@0:         for i = 1:nTestSets
wolffd@0:             size_sum = size_sum + size(simdata.partBinTrn{i,m}) / size(simdata.partBinTrn{i,end});
wolffd@0:         end
wolffd@0:         size_sum = size_sum / nTestSets;
wolffd@0:         
wolffd@0:         out.inctrain.trainfrac(:, m) = size_sum;
wolffd@0:         out.inctrain.dataPartition(:, m) = 0;
wolffd@0: 
wolffd@0:         % ---
wolffd@0:         % NOTE: the max value is important for debugging,
wolffd@0:         % especially when the maximal training success is reached
wolffd@0:         % in the middle of the data set
wolffd@0:         % ---
wolffd@0: %         out.inctrain.max_ok_test(:, m) = max(ok_test, 2);
wolffd@0:         out.inctrain.mean_ok_test(:, m) = mean(ok_test(:, ok_test(1,:) >=0), 2);
wolffd@0:         out.inctrain.var_ok_test(:, m) = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
wolffd@0:         out.inctrain.equal_test(m) = median(equal_test);
wolffd@0:         
wolffd@0:         out.inctrain.mean_ok_train(:, m) = mean(ok_train(:, ok_train(1,:) >=0), 2);
wolffd@0:         out.inctrain.var_ok_train(:, m) = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
wolffd@0:         
wolffd@0:         % ---
wolffd@0:         % TODO: DEBUG: this does not work correctly
wolffd@0:         % maybe thats also true for the above?
wolffd@0:         % ---
wolffd@0:         out.inctrain.mean_ok_notin_train(:, m) = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
wolffd@0:         out.inctrain.var_ok_notin_train(:, m) = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);
wolffd@0: 
wolffd@0:         diag.inctrain(m).ok_train = ok_train;
wolffd@0:         diag.inctrain(m).ok_test = ok_test;
wolffd@0:         diag.inctrain(m).ok_notin_train = ok_notin_train;
wolffd@0:         diag.inctrain(m).equal_test = equal_test;
wolffd@0:     end
wolffd@0:     
wolffd@0:     % ---
wolffd@0:     % save traditional information for full training set
wolffd@0:     % ---
wolffd@0:     if size(simdata.partBinTrn{1,m}) == size(simdata.partBinTrn{1,end});
wolffd@0: 
wolffd@0: %         out.max_ok_test = max(ok_test, 2);
wolffd@0:         out.mean_ok_test = mean(ok_test(:, ok_test(1,:) >=0), 2);
wolffd@0:         out.var_ok_test = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
wolffd@0:         out.equal_test = median(equal_test);
wolffd@0:                 
wolffd@0:         out.mean_ok_train = mean(ok_train(:, ok_train(1,:) >=0), 2);
wolffd@0:         out.var_ok_train = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
wolffd@0:         
wolffd@0:         % ---
wolffd@0:         % TODO: DEBUG: this does not work correctly
wolffd@0:         % ---
wolffd@0:         out.mean_ok_notin_train = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
wolffd@0:         out.var_ok_notin_train = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);
wolffd@0:         
wolffd@0:         % ---
wolffd@0:         % get winning measure
wolffd@0:         % we use the weighted winning measure if possible
wolffd@0:         % ---
wolffd@0:         if max(ok_test(2,:)) > 0
wolffd@0:             [~, best] = max(ok_test(2,:));
wolffd@0:         else
wolffd@0:             [~, best] = max(ok_test(1,:));
wolffd@0:         end
wolffd@0: 
wolffd@0:         diag.A = A;
wolffd@0:         diag.diag = dout;
wolffd@0: 
wolffd@0:         diag.ok_test = ok_test;
wolffd@0:         diag.equal_test = equal_test;
wolffd@0:         diag.ok_train = ok_train;
wolffd@0:         diag.ok_notin_train = ok_notin_train;
wolffd@0: 
wolffd@0:         % save some metric matrices
wolffd@0:         out.best_A = A{best};
wolffd@0:         out.best_diag = dout{best};
wolffd@0:         out.best_idx = best;   
wolffd@0:         
wolffd@0:     end
wolffd@0: end
wolffd@0: 
wolffd@0: % save parameters
wolffd@0: out.camirrev = camirversion();
wolffd@0: out.fparams = fparams;
wolffd@0: out.trainfun = trainfun;
wolffd@0: out.trainparams = trainparams;
wolffd@0: out.clip_ids = clips.id();
wolffd@0: out.dataPartition = [];
wolffd@0: out.Y = size(simdata);
wolffd@0: % ---
wolffd@0: % NOTE: this takes A LOT OF DISC SPACE
wolffd@0: % ---
wolffd@0: % out.Ytrain = Ytrain{end};
wolffd@0: % out.Ytest = Ytest{end};
wolffd@0: 
wolffd@0: % ---
wolffd@0: % save the diagostics data to disk
wolffd@0: % ---
wolffd@0: save(sprintf('runlog_%s.%s_results.mat',...
wolffd@0:         paramhash, paramhash_train),...
wolffd@0:         'out', 'diag');
wolffd@0: end