wolffd@0: function [p, med, avg, stats] = test_generic_significance_signrank(file1,run1,file2,run2, weighted, mode)
wolffd@0: % 
wolffd@0: % [p, med] = test_generic_get_signrank(input1, run1, input2, run2, weighted, mode)
wolffd@0: %
wolffd@0: % @param mode: string determining the way results are preprocessed: 
wolffd@0: %              '', 'avgbase', {'join_datasets',[sets2join]}           
wolffd@0: %
wolffd@0: % get wilcoxon signed rank of two test runs
wolffd@0: 
wolffd@0: if nargin < 5
wolffd@0:     weighted = 0;
wolffd@0: end
wolffd@0: if nargin < 6
wolffd@0:     avgbase = 0;
wolffd@0: end
wolffd@0: 
wolffd@0: 
wolffd@0: 
wolffd@0: % get individual cross-validation results
wolffd@0: [out1, ~, ~, indi1] = test_generic_display_results(file1, 0, 0);
wolffd@0: 
wolffd@0: [out2, ~, ~, indi2] = test_generic_display_results(file2, 0, 0);
wolffd@0: switch mode
wolffd@0:     case 'avgbase'
wolffd@0:     
wolffd@0:         %out2(run2).mean_ok_test
wolffd@0:         perf1 = indi1(run1).diag.ok_test(weighted+1,:);
wolffd@0:         perf2 = indi2(run2).diag.ok_test(weighted+1,:);
wolffd@0:         p2avg = mean(perf2);
wolffd@0:         perf_diffs = perf1 - p2avg;
wolffd@0: 
wolffd@0: 
wolffd@0:     case 'join_datasets'
wolffd@0:         %out2(run2).mean_ok_test
wolffd@0:         
wolffd@0:         % get the dataset indices which are to join
wolffd@0:         if isempty(run1) | (run1 < 1)
wolffd@0:             sets2join1 = 1:numel(indi1);
wolffd@0:         else
wolffd@0:             sets2join1 = run1;
wolffd@0:         end
wolffd@0:         if isempty(run2) | (run2 < 1)
wolffd@0:             sets2join2 = 1:numel(indi2);
wolffd@0:         else
wolffd@0:             sets2join2 = run2;
wolffd@0:         end
wolffd@0:         
wolffd@0:         perf1 = join_datasets(indi1,weighted,sets2join1);
wolffd@0:         perf2 = join_datasets(indi2,weighted,sets2join2);
wolffd@0:         perf_diffs = perf1 - perf2;
wolffd@0: 
wolffd@0:     otherwise
wolffd@0:         % get difference of individual results
wolffd@0:         perf_diffs = indi1(run1).diag.ok_test(weighted+1,:) - indi2(run2).diag.ok_test(weighted+1,:);       
wolffd@0: end
wolffd@0: 
wolffd@0: [p, h, stats] = signrank(perf_diffs );
wolffd@0: % [p, h] = signtest(perf1,perf2);
wolffd@0: % [h, p, stats] = kstest2(perf1,perf2);
wolffd@0: % [h2,p2] = ttest(perf_diffs);
wolffd@0:         
wolffd@0: % get median of performance difference
wolffd@0: med = median(perf_diffs);
wolffd@0: 
wolffd@0: % get median of performance difference
wolffd@0: avg = mean(perf_diffs);
wolffd@0: 
wolffd@0: % output interpretation
wolffd@0: if h && med > 0
wolffd@0:     fprintf('input 1 >> input 2 :)\n');
wolffd@0: elseif h && med < 0
wolffd@0:     fprintf('input 2 >> input 1 ;)\n');
wolffd@0: else
wolffd@0:     fprintf('no significant difference :( \n');
wolffd@0: end
wolffd@0: 
wolffd@0: 
wolffd@0: % ---
wolffd@0: % joins the test data to a single performance vector
wolffd@0: % ---
wolffd@0: function perf = join_datasets(indi,weighted, sets2join)
wolffd@0: perf = [];
wolffd@0: 
wolffd@0: for i = sets2join
wolffd@0:     perf = [perf indi(i).diag.ok_test(weighted+1,:)];
wolffd@0: end
wolffd@0:     
wolffd@0: