wolffd@0
|
1 function [out, stats] = write_mat_results_ISMIR13RBM_singletraining(dirin,fileout)
|
wolffd@0
|
2 % combine the test results from the directories supplied,
|
wolffd@0
|
3 % group them according to dataset parameter values
|
wolffd@0
|
4 % combine the test results from the directories supplied,
|
wolffd@0
|
5 % group them according to dataset parameter values
|
wolffd@0
|
6
|
wolffd@0
|
7 features = [];
|
wolffd@0
|
8 show = 1;
|
wolffd@0
|
9
|
wolffd@0
|
10 if nargin == 0
|
wolffd@0
|
11 dirin{1} = './';
|
wolffd@0
|
12 end
|
wolffd@0
|
13
|
wolffd@0
|
14 global comparison;
|
wolffd@0
|
15 global comparison_ids;
|
wolffd@0
|
16
|
wolffd@0
|
17 newout = [];
|
wolffd@0
|
18 thisdir = pwd;
|
wolffd@0
|
19 % loop through al lthe result directories and
|
wolffd@0
|
20 for diri = 1:numel(dirin)
|
wolffd@0
|
21
|
wolffd@0
|
22 % ---
|
wolffd@0
|
23 % go to directory and locate file
|
wolffd@0
|
24 % ---
|
wolffd@0
|
25 cd(dirin{diri});
|
wolffd@0
|
26
|
wolffd@0
|
27 u = dir();
|
wolffd@0
|
28 u = {u.name};
|
wolffd@0
|
29 [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1);
|
wolffd@0
|
30
|
wolffd@0
|
31 if numel(idx) < 1
|
wolffd@0
|
32 error 'This directory contains no valid test data';
|
wolffd@0
|
33 end
|
wolffd@0
|
34
|
wolffd@0
|
35 % just one or more tests in this folder?
|
wolffd@0
|
36 if exist('file','var') && isnumeric(file)
|
wolffd@0
|
37 cprint(1, 'loading one result file');
|
wolffd@0
|
38 file = u{idx(file)};
|
wolffd@0
|
39 data = load(file);
|
wolffd@0
|
40 sappend(out,data.out);
|
wolffd@0
|
41 else
|
wolffd@0
|
42 for filei = 1:numel(idx)
|
wolffd@0
|
43 cprint(1, 'loading result file %i of %i',filei, numel(idx));
|
wolffd@0
|
44 file = u{idx(filei)};
|
wolffd@0
|
45 data = load(file);
|
wolffd@0
|
46 newout = sappend(newout,data.out);
|
wolffd@0
|
47 end
|
wolffd@0
|
48 end
|
wolffd@0
|
49 % reset act directory
|
wolffd@0
|
50 cd(thisdir);
|
wolffd@0
|
51 end
|
wolffd@0
|
52
|
wolffd@0
|
53 % ---
|
wolffd@0
|
54 % filter according to training parameter C
|
wolffd@0
|
55 %
|
wolffd@0
|
56 % NOTE :if we don't filter by C, we get strong overfitting with training
|
wolffd@0
|
57 % success > 96 % and test set performance aorund 65 %
|
wolffd@0
|
58 % ---
|
wolffd@0
|
59 cs = zeros(numel(newout),1);
|
wolffd@0
|
60 for i=1:numel(newout)
|
wolffd@0
|
61 cs(i) = newout(i).trainparams.C;
|
wolffd@0
|
62 end
|
wolffd@0
|
63 cvals = unique(cs);
|
wolffd@0
|
64
|
wolffd@0
|
65 for ci=1:numel(cvals)
|
wolffd@0
|
66 valididx = find(cs == cvals(ci));
|
wolffd@0
|
67 filteredout = newout(valididx);
|
wolffd@0
|
68
|
wolffd@0
|
69 % ---
|
wolffd@0
|
70 % get parameter statistics
|
wolffd@0
|
71 % ---
|
wolffd@0
|
72 stats = test_generic_display_param_influence(filteredout, show);
|
wolffd@0
|
73
|
wolffd@0
|
74 % get maximal values for each test set bin
|
wolffd@0
|
75 % ---
|
wolffd@0
|
76 % trainparams.dataset contains sets which have each only one bin of the
|
wolffd@0
|
77 % ismir testsets
|
wolffd@0
|
78 % ---
|
wolffd@0
|
79 max_idx = [stats.trainparams.dataset.mean_ok_train.max_idx];
|
wolffd@0
|
80 ok_test = zeros(2, numel(max_idx));
|
wolffd@0
|
81 ok_train = zeros(2, numel(max_idx));
|
wolffd@0
|
82 ok_config = [];
|
wolffd@0
|
83 % cycle over all test sets and save best result
|
wolffd@0
|
84 for i=1:numel(max_idx)
|
wolffd@0
|
85 ok_test(:,i) = filteredout(max_idx(i)).mean_ok_test;
|
wolffd@0
|
86 ok_train(:,i) = filteredout(max_idx(i)).mean_ok_train;
|
wolffd@0
|
87 ok_config = sappend(ok_config,struct('trainparams',filteredout(max_idx(i)).trainparams, ...
|
wolffd@0
|
88 'fparams',filteredout(max_idx(i)).fparams));
|
wolffd@0
|
89 end
|
wolffd@0
|
90 % save the stuff
|
wolffd@0
|
91 out(ci).mean_ok_test = mean(ok_test,2);
|
wolffd@0
|
92 out(ci).var_ok_test = var(ok_test,0,2);
|
wolffd@0
|
93 out(ci).mean_ok_train = mean(ok_train,2);
|
wolffd@0
|
94 out(ci).var_ok_train = var(ok_train,0,2);
|
wolffd@0
|
95 out(ci).trainparams.C = cvals(ci);
|
wolffd@0
|
96 out(ci).ok_config = ok_config;
|
wolffd@0
|
97 out(ci).ok_test = ok_test;
|
wolffd@0
|
98 out(ci).ok_train = ok_train;
|
wolffd@0
|
99 end
|
wolffd@0
|
100
|
wolffd@0
|
101 % ---
|
wolffd@0
|
102 % show results for different C
|
wolffd@0
|
103 % ---
|
wolffd@0
|
104 if numel([out.mean_ok_test]) > 1 && show
|
wolffd@0
|
105
|
wolffd@0
|
106 % plot means % plot std = sqrt(var) % plot training results
|
wolffd@0
|
107 figure;
|
wolffd@0
|
108 boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]);
|
wolffd@0
|
109 title (sprintf('Performance for all configs'));
|
wolffd@0
|
110 end
|
wolffd@0
|
111
|
wolffd@0
|
112 % ---
|
wolffd@0
|
113 % write max. test success
|
wolffd@0
|
114 % ---
|
wolffd@0
|
115 mean_ok_test = [out.mean_ok_test];
|
wolffd@0
|
116 [val, idx] = max(mean_ok_test(1,:));
|
wolffd@0
|
117 if show
|
wolffd@0
|
118 fprintf(' --- Maximal test set success: nr. %d, %3.2f percent. --- \n', idx, val * 100)
|
wolffd@0
|
119 end
|
wolffd@0
|
120
|
wolffd@0
|
121 % save
|
wolffd@0
|
122 save([hash(strcat(dirin{:}),'md5') '_summary'], 'out');
|
wolffd@0
|
123
|
wolffd@0
|
124 end
|
wolffd@0
|
125
|
wolffd@0
|
126
|
wolffd@0
|
127
|
wolffd@0
|
128 function boxplot(mean, std, train);
|
wolffd@0
|
129
|
wolffd@0
|
130 bar([train; mean]', 1.5);
|
wolffd@0
|
131 hold on;
|
wolffd@0
|
132 errorbar(1:size(mean,2), mean(1,:), std(1,:),'.');
|
wolffd@0
|
133 % plot(train,'rO');
|
wolffd@0
|
134 colormap(spring);
|
wolffd@0
|
135 axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]);
|
wolffd@0
|
136 end
|