wolffd@0
|
1 function [out, stats] = write_mat_results_ISMIR13RBM(dirin,fileout)
|
wolffd@0
|
2 % combine the test results from the directories supplied,
|
wolffd@0
|
3 % and recombine the bins to a coherent dataset
|
wolffd@0
|
4 %
|
wolffd@0
|
5 % allows for distinguishing different hidden State numbers
|
wolffd@0
|
6
|
wolffd@0
|
7 % folders = {'130505_svm_disttest_subsconv_rbm_largegrid_21_r2385','130505_svm_disttest_subsconv_rbm_largegrid_22_r2385','130505_svm_disttest_subsconv_rbm_largegrid_23_r2385','130505_svm_disttest_subsconv_rbm_largegrid_24_r2385','130506_svm_disttest_subsconv_rbm_largegrid_25_r2385'};
|
wolffd@0
|
8
|
wolffd@0
|
9
|
wolffd@0
|
10
|
wolffd@0
|
11 features = [];
|
wolffd@0
|
12 show = 1;
|
wolffd@0
|
13
|
wolffd@0
|
14 if nargin == 0
|
wolffd@0
|
15 dirin{1} = './';
|
wolffd@0
|
16 end
|
wolffd@0
|
17
|
wolffd@0
|
18 global comparison;
|
wolffd@0
|
19 global comparison_ids;
|
wolffd@0
|
20
|
wolffd@0
|
21 newout = [];
|
wolffd@0
|
22 thisdir = pwd;
|
wolffd@0
|
23 % loop through al lthe result directories and
|
wolffd@0
|
24 for diri = 1:numel(dirin)
|
wolffd@0
|
25
|
wolffd@0
|
26 % ---
|
wolffd@0
|
27 % go to directory and locate file
|
wolffd@0
|
28 % ---
|
wolffd@0
|
29 cd(dirin{diri});
|
wolffd@0
|
30
|
wolffd@0
|
31 u = dir();
|
wolffd@0
|
32 u = {u.name};
|
wolffd@0
|
33 [idx, strpos] = substrcellfind(u, '_finalresults.mat', 1);
|
wolffd@0
|
34
|
wolffd@0
|
35 if numel(idx) < 1
|
wolffd@0
|
36 error 'This directory contains no valid test data';
|
wolffd@0
|
37 end
|
wolffd@0
|
38
|
wolffd@0
|
39 % just one or more tests in this folder?
|
wolffd@0
|
40 if exist('file','var') && isnumeric(file)
|
wolffd@0
|
41 cprint(1, 'loading one result file');
|
wolffd@0
|
42 file = u{idx(file)};
|
wolffd@0
|
43 data = load(file);
|
wolffd@0
|
44 sappend(out,data.out);
|
wolffd@0
|
45 else
|
wolffd@0
|
46 for filei = 1:numel(idx)
|
wolffd@0
|
47 cprint(1, 'loading result file %i of %i',filei, numel(idx));
|
wolffd@0
|
48 file = u{idx(filei)};
|
wolffd@0
|
49 data = load(file);
|
wolffd@0
|
50 newout = sappend(newout,data.out);
|
wolffd@0
|
51 end
|
wolffd@0
|
52 end
|
wolffd@0
|
53 % reset act directory
|
wolffd@0
|
54 cd(thisdir);
|
wolffd@0
|
55 end
|
wolffd@0
|
56
|
wolffd@0
|
57
|
wolffd@0
|
58 % ---
|
wolffd@0
|
59 % filter C values!
|
wolffd@0
|
60 % ---
|
wolffd@0
|
61 allcs = zeros(numel(newout),1);
|
wolffd@0
|
62 for i=1:numel(newout)
|
wolffd@0
|
63 allcs(i) = newout(i).trainparams.C;
|
wolffd@0
|
64 end
|
wolffd@0
|
65 valididx = find(allcs == 1); % select C!
|
wolffd@0
|
66 newout = newout(valididx);
|
wolffd@0
|
67
|
wolffd@0
|
68
|
wolffd@0
|
69 % bundle all datasets
|
wolffd@0
|
70 fout = sameparamsubset(newout, 'dataset','');
|
wolffd@0
|
71 out = [];
|
wolffd@0
|
72 for ci=1:numel(fout)
|
wolffd@0
|
73 filteredout = fout{ci};
|
wolffd@0
|
74
|
wolffd@0
|
75
|
wolffd@0
|
76 ok_test = zeros(2, numel(filteredout));
|
wolffd@0
|
77 ok_train = zeros(2, numel(filteredout));
|
wolffd@0
|
78 ok_config = [];
|
wolffd@0
|
79
|
wolffd@0
|
80
|
wolffd@0
|
81 tmpout = filteredout(1);
|
wolffd@0
|
82 % cycle over all test sets and get new means
|
wolffd@0
|
83 for i=1:numel(filteredout)
|
wolffd@0
|
84 ok_test(:,i) = filteredout(i).mean_ok_test;
|
wolffd@0
|
85 ok_train(:,i) = filteredout(i).mean_ok_train;
|
wolffd@0
|
86 end
|
wolffd@0
|
87
|
wolffd@0
|
88 % save the stuff
|
wolffd@0
|
89 tmpout.mean_ok_test = mean(ok_test,2);
|
wolffd@0
|
90 tmpout.var_ok_test = var(ok_test,0,2);
|
wolffd@0
|
91 tmpout.mean_ok_train = mean(ok_train,2);
|
wolffd@0
|
92 tmpout.var_ok_train = var(ok_train,0,2);
|
wolffd@0
|
93
|
wolffd@0
|
94 tmpout.ok_test = ok_test;
|
wolffd@0
|
95 tmpout.ok_train = ok_train;
|
wolffd@0
|
96
|
wolffd@0
|
97 % put it in output structure
|
wolffd@0
|
98 out = sappend(out,tmpout);
|
wolffd@0
|
99 end
|
wolffd@0
|
100
|
wolffd@0
|
101 % ---
|
wolffd@0
|
102 % show results
|
wolffd@0
|
103 % ---
|
wolffd@0
|
104 if numel([out.mean_ok_test]) > 1 && show
|
wolffd@0
|
105
|
wolffd@0
|
106 % plot means % plot std = sqrt(var) % plot training results
|
wolffd@0
|
107 figure;
|
wolffd@0
|
108 boxplot([out.mean_ok_test], sqrt([out.var_ok_test]), [out.mean_ok_train]);
|
wolffd@0
|
109 title (sprintf('Performance for all configs'));
|
wolffd@0
|
110 end
|
wolffd@0
|
111
|
wolffd@0
|
112
|
wolffd@0
|
113
|
wolffd@0
|
114 % ---
|
wolffd@0
|
115 % write max. test success
|
wolffd@0
|
116 % ---
|
wolffd@0
|
117 mean_ok_test = [out.mean_ok_test];
|
wolffd@0
|
118 [val, idx] = max(mean_ok_test(1,:));
|
wolffd@0
|
119 if show
|
wolffd@0
|
120 fprintf(' --- Maximal test set success: nr. %d, %3.2f percent. train result %3.2f percent. --- \n', idx, val * 100,out(idx).mean_ok_train(1,:)*100)
|
wolffd@0
|
121 end
|
wolffd@0
|
122
|
wolffd@0
|
123
|
wolffd@0
|
124 % ---
|
wolffd@0
|
125 % write max. test/train success
|
wolffd@0
|
126 % ---
|
wolffd@0
|
127 mean_ok_train = [out.mean_ok_train];
|
wolffd@0
|
128 [val, idx] = max(mean_ok_train(1,:));
|
wolffd@0
|
129 if show
|
wolffd@0
|
130 fprintf(' --- Maximal train set success: nr. %d, %3.2f percent, test result %3.2f percent. --- \n', idx, val * 100,out(idx).mean_ok_test(1,:)*100)
|
wolffd@0
|
131 end
|
wolffd@0
|
132
|
wolffd@0
|
133 % save this summary
|
wolffd@0
|
134 save([hash(strcat(dirin{:}),'md5') '_summary'], 'out');
|
wolffd@0
|
135
|
wolffd@0
|
136
|
wolffd@0
|
137
|
wolffd@0
|
138 % ---
|
wolffd@0
|
139 % build index of these feature configurations without num_hid
|
wolffd@0
|
140 % ---
|
wolffd@0
|
141 [fout2,param_hash,idxhash] = sameparamsubset(out, '','rbm_hidNum');
|
wolffd@0
|
142 for ci = 1:numel(fout2)
|
wolffd@0
|
143
|
wolffd@0
|
144 % search for the training param index
|
wolffd@0
|
145 found = find(idxhash{ci}== idx);
|
wolffd@0
|
146 if ~isempty(found)
|
wolffd@0
|
147 out_withsameconfig = fout2{ci};
|
wolffd@0
|
148 break;
|
wolffd@0
|
149 end
|
wolffd@0
|
150 end
|
wolffd@0
|
151
|
wolffd@0
|
152
|
wolffd@0
|
153 end
|
wolffd@0
|
154
|
wolffd@0
|
155 function [out, param_hash,idx] = sameparamsubset(in, ignoret,ignoref)
|
wolffd@0
|
156 % ---
|
wolffd@0
|
157 % build index of all existing configurations
|
wolffd@0
|
158 % ---
|
wolffd@0
|
159 param_hash = cell(numel(in),1);
|
wolffd@0
|
160 for i=1:numel(in)
|
wolffd@0
|
161 params = struct('trainparams',in(i).trainparams, ...
|
wolffd@0
|
162 'fparams',in(i).fparams);
|
wolffd@0
|
163
|
wolffd@0
|
164 % remove the dataset param
|
wolffd@0
|
165 if ~isempty(ignoret)
|
wolffd@0
|
166 params.trainparams = rmfield(params.trainparams,ignoret);
|
wolffd@0
|
167 end
|
wolffd@0
|
168 if ~isempty(ignoref)
|
wolffd@0
|
169 params.fparams = rmfield(params.fparams,ignoref);
|
wolffd@0
|
170 end
|
wolffd@0
|
171
|
wolffd@0
|
172 phash = hash(xml_format(params),'md5');
|
wolffd@0
|
173 param_hash{i} = phash;
|
wolffd@0
|
174 end
|
wolffd@0
|
175
|
wolffd@0
|
176 % ---
|
wolffd@0
|
177 % recombine the data for different datasets!
|
wolffd@0
|
178 % ---
|
wolffd@0
|
179 cvals = unique(param_hash);
|
wolffd@0
|
180
|
wolffd@0
|
181 out = {};
|
wolffd@0
|
182 for ci=1:numel(cvals)
|
wolffd@0
|
183 idx{ci} = strcellfind(param_hash,cvals(ci),1);
|
wolffd@0
|
184 out{ci} = in(idx{ci});
|
wolffd@0
|
185 end
|
wolffd@0
|
186
|
wolffd@0
|
187 end
|
wolffd@0
|
188
|
wolffd@0
|
189 function boxplot(mean, std, train);
|
wolffd@0
|
190
|
wolffd@0
|
191 bar([train; mean]', 1.5);
|
wolffd@0
|
192 hold on;
|
wolffd@0
|
193 errorbar(1:size(mean,2), mean(1,:), std(1,:),'.');
|
wolffd@0
|
194 % plot(train,'rO');
|
wolffd@0
|
195 colormap(spring);
|
wolffd@0
|
196 axis([0 size(mean,2)+1 max(0, min(min([train mean] - 0.1))) max(max([train mean] + 0.1))]);
|
wolffd@0
|
197 end
|