view core/magnatagatune/tests_evals/do_test_rounds.m @ 0:e9a9cd732c1e tip

first hg version after svn

function [out]= do_test_rounds(trainfun, X, simdata, trainparams, fparams,...
            paramhash, paramhash_train, clips)

% ---
% DEBUG: we mix up the training set
% ---
% simdata = mixup(simdata);

if isfield(simdata, 'clip_type');
    clip_type = simdata.clip_type;
else
    clip_type = 'MTTClip';
end

nTestSets = size(simdata.partBinTst, 2); % num cv bins
ntrainsizes = size(simdata.partBinTrn, 2); % num increases of training

for m = 1:ntrainsizes

    ok_train = zeros(2, nTestSets);
    ok_test = zeros(2, nTestSets);
    equal_test = zeros(1, nTestSets);
    ok_notin_train = zeros(2, nTestSets);
    
%     A = cell(nTestSets,1);
%     dout = cell(nTestSets,1);
%     clips_train = cell(nTestSets,1);
%     clips_test = cell(nTestSets,1);
%     clips_notin_train = cell(nTestSets,1);
%     Y_notin_train = cell(nTestSets,1);
%     Ytrain = cell(nTestSets,1);
%     Ytest = cell(nTestSets,1);
    % parfor
    for k = 1:nTestSets


        % runlog mlr
        try
            
            % ---
            % Get the training constraints and features for this round
            % ---
            % DEBUG: the similarity data in Ytrain and Ytest seems correct.
            [clips_train{k}, Xtrain, Ytrain{k}] ...
                = get_data_compact(clips, X, simdata.partBinTrn{k,m});
            Ytest{k}={};
            
            % training step
            [A{k}, dout{k}] = feval(trainfun, Xtrain, Ytrain{k}, trainparams);

            % ---
            % test step
            % TODO: the distmeasure object could be created by the wrapper! 
            % ---
            if isfield(dout{k},'interpreter');
                interpreter = str2func(dout{k}.interpreter);
            else
                % only for backward compability
                % warning ('legacy implementation of dist measure');
                 if isnumeric(A{k}) 
                    % mahalanobis case

                    % special delta mahalanobis
                    interpreter = str2func('DistMeasureMahal');
                else 
                    % neural network case: A{k} is a neural net object
                    interpreter = str2func('DistMeasureGeneric');
                end
            end
            
            if isfield(trainparams,'deltafun')
                % special delta 
                diss = feval(interpreter,clips, A{k}, X, str2func(trainparams.deltafun), trainparams.deltafun_params);
            else
                % standard 
                % ---
                % TODO: the default delta is different between 
                % similarity measures. except for the mahalmeasure
                % this should be specified
                % ---
                diss = feval(interpreter, clips, A{k}, X);
            end
            
            % test training data 
             [ok_train(:,k)] = metric_fulfills_ranking...
                (diss, Ytrain{k}, feval(clip_type,clips_train{k}));
            
            % get test data  
            [clips_test{k}, Xtest, Ytest{k}] ...
                = get_data_compact(clips, X, simdata.partBinTst{k});
            
           % diss = DistMeasureMahal(feval(clip_type,clips_test{k}), A{k}, Xtest);
            % test test data
            [ok_test(:,k), equal_test(k)] = metric_fulfills_ranking...
                (diss, Ytest{k}, feval(clip_type,clips_test{k}));
            cprint(3,'%2.2f %2.2f fold performance', ok_test(:,k));
            
            % ---
            % extra diag for MLR
            % TODO: make this wrappeable
            % ---
            if isequal(trainfun, @mlr_wrapper)
                dout{k}.mprperf =  mlr_test(A{k}, 0, Xtrain, Ytrain{k}(:,1:2), Xtest, Ytest{k}(:,1:2)) ;
            end
            
            % --- 
            % this gives data for the unused training set remainders
            % ---
            if isfield(simdata,'partBinNoTrn') 
                if ~isempty(simdata.partBinNoTrn{k,m})
                [clips_notin_train{k}, X_notin_train, Y_notin_train{k}] ...
                    = get_data_compact(clips, X, simdata.partBinNoTrn{k,m});
           
                % test unused training data
                [ok_notin_train(:,k), equal_test(k)] = metric_fulfills_ranking...
                    (diss, Y_notin_train{k}, feval(clip_type,clips_notin_train{k}));
                
                % what to do if there is no data ? 
                else
                    ok_notin_train(:,k) = -1;
                end
            else
                ok_notin_train(:,k) = -1;
            end

        catch err
            
            % ---
            % in case training or test fails
            % ---
            print_error(err);
            
            A{k} = [];
            dout{k} = -1;

            ok_test(:,k) = -1;
            ok_train(:,k) = -1;
            ok_notin_train(:,k) = -1;
            equal_test(k) = -1;
            
            % ---
            % save feature, system and data configuration
            % and indicate failure
            % ---
            xml_save(sprintf('runlog_%s.%s_trainparam.xml',...
                paramhash, paramhash_train), trainparams);
            xml_save(sprintf('runlog_%s.%s_err.xml',...
                paramhash, paramhash_train), print_error(err));
        end
    end

    if ~(ntrainsizes == 1)
        
        % save elaborate testing data
        size_sum = 0;
        for i = 1:nTestSets
            size_sum = size_sum + size(simdata.partBinTrn{i,m}) / size(simdata.partBinTrn{i,end});
        end
        size_sum = size_sum / nTestSets;
        
        out.inctrain.trainfrac(:, m) = size_sum;
        out.inctrain.dataPartition(:, m) = 0;

        % ---
        % NOTE: the max value is important for debugging,
        % especially when the maximal training success is reached
        % in the middle of the data set
        % ---
%         out.inctrain.max_ok_test(:, m) = max(ok_test, 2);
        out.inctrain.mean_ok_test(:, m) = mean(ok_test(:, ok_test(1,:) >=0), 2);
        out.inctrain.var_ok_test(:, m) = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
        out.inctrain.equal_test(m) = median(equal_test);
        
        out.inctrain.mean_ok_train(:, m) = mean(ok_train(:, ok_train(1,:) >=0), 2);
        out.inctrain.var_ok_train(:, m) = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
        
        % ---
        % TODO: DEBUG: this does not work correctly
        % maybe thats also true for the above?
        % ---
        out.inctrain.mean_ok_notin_train(:, m) = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
        out.inctrain.var_ok_notin_train(:, m) = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);

        diag.inctrain(m).ok_train = ok_train;
        diag.inctrain(m).ok_test = ok_test;
        diag.inctrain(m).ok_notin_train = ok_notin_train;
        diag.inctrain(m).equal_test = equal_test;
    end
    
    % ---
    % save traditional information for full training set
    % ---
    if size(simdata.partBinTrn{1,m}) == size(simdata.partBinTrn{1,end});

%         out.max_ok_test = max(ok_test, 2);
        out.mean_ok_test = mean(ok_test(:, ok_test(1,:) >=0), 2);
        out.var_ok_test = var(ok_test(:, ok_test(1,:) >=0), 0, 2);
        out.equal_test = median(equal_test);
                
        out.mean_ok_train = mean(ok_train(:, ok_train(1,:) >=0), 2);
        out.var_ok_train = var(ok_train(:, ok_train(1,:) >=0), 0, 2);
        
        % ---
        % TODO: DEBUG: this does not work correctly
        % ---
        out.mean_ok_notin_train = mean(ok_notin_train(:, ok_notin_train(1,:) >=0), 2);
        out.var_ok_notin_train = var(ok_notin_train(:, ok_notin_train(1,:) >=0), 0, 2);
        
        % ---
        % get winning measure
        % we use the weighted winning measure if possible
        % ---
        if max(ok_test(2,:)) > 0
            [~, best] = max(ok_test(2,:));
        else
            [~, best] = max(ok_test(1,:));
        end

        diag.A = A;
        diag.diag = dout;

        diag.ok_test = ok_test;
        diag.equal_test = equal_test;
        diag.ok_train = ok_train;
        diag.ok_notin_train = ok_notin_train;

        % save some metric matrices
        out.best_A = A{best};
        out.best_diag = dout{best};
        out.best_idx = best;   
        
    end
end

% save parameters
out.camirrev = camirversion();
out.fparams = fparams;
out.trainfun = trainfun;
out.trainparams = trainparams;
out.clip_ids = clips.id();
out.dataPartition = [];
out.Y = size(simdata);
% ---
% NOTE: this takes A LOT OF DISC SPACE
% ---
% out.Ytrain = Ytrain{end};
% out.Ytest = Ytest{end};

% ---
% save the diagostics data to disk
% ---
save(sprintf('runlog_%s.%s_results.mat',...
        paramhash, paramhash_train),...
        'out', 'diag');
end