Mercurial > hg > camir-aes2014
comparison core/tools/test_generic_significance_signrank.m @ 0:e9a9cd732c1e tip
first hg version after svn
author | wolffd |
---|---|
date | Tue, 10 Feb 2015 15:05:51 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:e9a9cd732c1e |
---|---|
1 function [p, med, avg, stats] = test_generic_significance_signrank(file1,run1,file2,run2, weighted, mode) | |
2 % | |
3 % [p, med] = test_generic_get_signrank(input1, run1, input2, run2, weighted, mode) | |
4 % | |
5 % @param mode: string determining the way results are preprocessed: | |
6 % '', 'avgbase', {'join_datasets',[sets2join]} | |
7 % | |
8 % get wilcoxon signed rank of two test runs | |
9 | |
10 if nargin < 5 | |
11 weighted = 0; | |
12 end | |
13 if nargin < 6 | |
14 avgbase = 0; | |
15 end | |
16 | |
17 | |
18 | |
19 % get individual cross-validation results | |
20 [out1, ~, ~, indi1] = test_generic_display_results(file1, 0, 0); | |
21 | |
22 [out2, ~, ~, indi2] = test_generic_display_results(file2, 0, 0); | |
23 switch mode | |
24 case 'avgbase' | |
25 | |
26 %out2(run2).mean_ok_test | |
27 perf1 = indi1(run1).diag.ok_test(weighted+1,:); | |
28 perf2 = indi2(run2).diag.ok_test(weighted+1,:); | |
29 p2avg = mean(perf2); | |
30 perf_diffs = perf1 - p2avg; | |
31 | |
32 | |
33 case 'join_datasets' | |
34 %out2(run2).mean_ok_test | |
35 | |
36 % get the dataset indices which are to join | |
37 if isempty(run1) | (run1 < 1) | |
38 sets2join1 = 1:numel(indi1); | |
39 else | |
40 sets2join1 = run1; | |
41 end | |
42 if isempty(run2) | (run2 < 1) | |
43 sets2join2 = 1:numel(indi2); | |
44 else | |
45 sets2join2 = run2; | |
46 end | |
47 | |
48 perf1 = join_datasets(indi1,weighted,sets2join1); | |
49 perf2 = join_datasets(indi2,weighted,sets2join2); | |
50 perf_diffs = perf1 - perf2; | |
51 | |
52 otherwise | |
53 % get difference of individual results | |
54 perf_diffs = indi1(run1).diag.ok_test(weighted+1,:) - indi2(run2).diag.ok_test(weighted+1,:); | |
55 end | |
56 | |
57 [p, h, stats] = signrank(perf_diffs ); | |
58 % [p, h] = signtest(perf1,perf2); | |
59 % [h, p, stats] = kstest2(perf1,perf2); | |
60 % [h2,p2] = ttest(perf_diffs); | |
61 | |
62 % get median of performance difference | |
63 med = median(perf_diffs); | |
64 | |
65 % get median of performance difference | |
66 avg = mean(perf_diffs); | |
67 | |
68 % output interpretation | |
69 if h && med > 0 | |
70 fprintf('input 1 >> input 2 :)\n'); | |
71 elseif h && med < 0 | |
72 fprintf('input 2 >> input 1 ;)\n'); | |
73 else | |
74 fprintf('no significant difference :( \n'); | |
75 end | |
76 | |
77 | |
78 % --- | |
79 % joins the test data to a single performance vector | |
80 % --- | |
81 function perf = join_datasets(indi,weighted, sets2join) | |
82 perf = []; | |
83 | |
84 for i = sets2join | |
85 perf = [perf indi(i).diag.ok_test(weighted+1,:)]; | |
86 end | |
87 | |
88 |