Mercurial > hg > camir-aes2014

comparison core/magnatagatune/makro_cupaper12_get_significance.m @ 0:e9a9cd732c1e tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first hg version after svn
author wolffd
date Tue, 10 Feb 2015 15:05:51 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:e9a9cd732c1e
1 % makro_get_significance_cupaper
2 %%
3 % ------------------------------ Algorithms Compared id-sampling
4 % ---
5 'mlr vs euclidean, mlr all average feat, ID-sampling'
6 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1a58077f1232c33b787b661039df107d_finalresults';
7
8 % euclidean average feat - TODO: which sampling is this ?
9 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
10 % ---
11 % NOTE: all 4 test-set runs have exactly the same result
12 % ---
13 run1 = -1;
14 run2 = -1;
15 mode = 'join_datasets'; % '', 'avgbase', 'join_datasets'
16 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run1,0,mode);
17 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
18
19
20 %%
21 % ---
22 'SVM vs euclidean, SVM all average feat, ID-sampling'
23 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults';
24
25 % euclidean average feat - TODO: which sampling is this ?
26 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
27
28 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode);
29 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
30
31 %%
32 % ------------------------------ Algorithms Compared TD-sampling
33 % ---
34 'mlr unclustered vs euclidean unclustered, mlr all average feat TD-sampling'
35 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_c364cb0803822d55f2940656c44b184d_finalresults';
36
37 % euclidean average feat TD-sampling
38 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_72303bfa642aad872665dee7a3b1e28c_finalresults';
39
40 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode);
41 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
42
43 %%
44 % ---
45 'SVM unclustered vs euclidean unclustered, SVM all average feat TD-sampling'
46 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3434f0534fa910b26bbf927c65a7fb74_finalresults';
47
48 % euclidean average feat TD-sampling
49 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_72303bfa642aad872665dee7a3b1e28c_finalresults';
50 % ---
51
52 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode);
53 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
54
55
56 %%
57 % ------------------------------ Features Compared
58 'All 12-dim PCA Features compared'
59 % ---
60 namePCA12 = {'AcousticPCA12', ...
61 'Slaney08PCA12', ...
62 'TimbrePCA12', ...
63 'ChromaPCA12', ...
64 'AllPCA12', ...
65 'GenrePCA12'};
66 % this is the index how the feature types appear in the paper
67 paperidx = [4 3 2 6 1 5];
68
69 filePCA12 = {'runlog_a18bd2111694ac59c9ba0a6810121796_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
70 'runlog_37e47c187886f73ec9a7d8dc24a84a52_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
71 'runlog_4c6787b403a07f5faf1ec26e891da4fa_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
72 'runlog_c5566f74e6a0d00b50f5eea05fdacfee_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
73 'runlog_e2c22696e7af9e7eea1fa1fd10a1f785_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
74 'runlog_efc6e5e9c56291cd1744092a1c59a293_1841892e9df07039bbe4c3a55d11026a_finalresults'};
75
76
77 namePCA12 = namePCA12(paperidx);
78 filePCA12 = filePCA12(paperidx);
79
80 p = zeros(numel(namePCA12),numel(namePCA12));
81 med = zeros(numel(namePCA12),numel(namePCA12));
82 avg = zeros(numel(namePCA12),numel(namePCA12));
83 for i=1:numel(name)
84 for j = 1:i-1
85 [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(filePCA12{i},-1,filePCA12{j},-1,0,mode);
86 end
87 % p(i,i) = 0.5;
88 end
89 % p = p + p';
90 imagesc(p);
91 colormap(hot)
92 axis xy
93 set(gca,'XTick',1:numel(namePCA12), 'XTickLabel', namePCA12);
94 set(gca,'YTick',1:numel(namePCA12), 'YTickLabel', namePCA12);
95 matrix2latex(p,'%1.3f')
96
97 %%
98 'All 52-dim PCA Features compared'
99 % ---
100 namePCA52 = {'GenrePCA52', ...
101 'AllPCA52', ...
102 'TimbrePCA52', ...
103 'AcousticPCA52', ...
104 'ChromaPCA52' ...
105 };
106 % this is the index how the feature types appear in the paper
107 paperidx = [5 3 1 4 2];
108
109 filePCA52 = {'runlog_3cbf4759cf58af0728aaab0b5f2660e3_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
110 'runlog_7d5fafec0dc504215acc8cb7a9202a56_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
111 'runlog_a3c2c0a5742a42fd54497e69b8f44e8d_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
112 'runlog_c7164074206998aa184538bedcfdcf2f_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
113 'runlog_efbf7c8e75ae154c2f192acd08fbdcbc_1841892e9df07039bbe4c3a55d11026a_finalresults' ...
114 };
115
116
117 namePCA52 = namePCA52(paperidx);
118 filePCA52 = filePCA52(paperidx);
119
120 p = zeros(numel(namePCA52),numel(namePCA52));
121 med = zeros(numel(namePCA52),numel(namePCA52));
122 avg = zeros(numel(namePCA52),numel(namePCA52));
123 for i=1:numel(name)
124 for j = 1:i-1
125 [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(...
126 filePCA52{i},run1,filePCA52{j},run2,0,mode);
127 end
128 % p(i,i) = 0.5;
129 end
130 % p = p + p';
131 imagesc(p);
132 colormap(hot)
133 axis xy
134 set(gca,'XTick',1:numel(namePCA52), 'XTickLabel', namePCA52);
135 set(gca,'YTick',1:numel(namePCA52), 'YTickLabel', namePCA52);
136 matrix2latex(p,'%1.3f')
137
138 %%
139 'All full-dim Features compared (SVM)'
140 % ---
141 name = {'4Chroma', ...
142 '1Chroma', ...
143 '4Timbre', ...
144 '1Timbre', ...
145 '1Acoustic', ...
146 'Genre', ...
147 'Slaney08', ...
148 '1All', ...
149 '4All', ...
150 '4Acoustic' ...
151 };
152
153 % this is the index how the feature types appear in the paper
154 % Features & Chroma(1/4) & Timbre(1/4) & Slaney08 & Genre & Comb. Audio (1/4) & Comb. All(1/4) \\
155 paperidx = [2 1 4 3 7 6 5 10 8 9];
156
157 file = {'runlog_20a2f6a0f20f488e9386ebb8c5026fcf_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
158 'runlog_20a2f6a0f20f488e9386ebb8c5026fcf_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
159 'runlog_3154f36c34c18f60218c5d3f0c0b5931_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
160 'runlog_3154f36c34c18f60218c5d3f0c0b5931_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
161 'runlog_31981d48dd0d25564ef3c2b3ca650b3b_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
162 'runlog_37867d3b5bd4c74b7b548732b80fb947_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
163 'runlog_f52d37439805ac4edc70b0432281abc3_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
164 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
165 'runlog_800d97be9ef6274dc3bbe6b9be2406a6_1a58077f1232c33b787b661039df107d_finalresults', ...
166 'runlog_cf5a61cca09e2a3182b794b70ee1ab91_1841892e9df07039bbe4c3a55d11026a_finalresults' ...
167 };
168
169 sets2join = {[1:4],[5:8],[1:4],[5:8],[],[],[],[],[],[]};
170
171 name = name(paperidx);
172 file = file(paperidx);
173
174 p = zeros(numel(name),numel(name));
175 med = zeros(numel(name),numel(name));
176 avg = zeros(numel(name),numel(name));
177 for i=1:numel(name)
178 for j = 1:i-1
179 [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(...
180 file{i},sets2join{i},file{j},sets2join{j},0,mode);
181 end
182 % p(i,i) = 0.5;
183 end
184 p = flipud(p);
185 % p = p + p';
186 imagesc(p);
187 colormap(hot)
188 axis ij
189 set(gca,'XTick',1:numel(name), 'XTickLabel', name);
190 set(gca,'YTick',1:numel(name), 'YTickLabel', fliplr(name));
191 matrix2latex(p,'%1.3f')
192
193 %%
194 'Comparing Feature dimensions and PCA effect on combined all features'
195 % ---
196 % 'SVM all average feat, ID-sampling'
197 allavg = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults';
198
199 % 'SVM 4 cluster feat, ID-sampling'
200 allfourcluster = 'runlog_800d97be9ef6274dc3bbe6b9be2406a6_1a58077f1232c33b787b661039df107d_finalresults';
201
202 % 'SVM 12dim feat, ID-sampling'
203 pca12 = 'runlog_e2c22696e7af9e7eea1fa1fd10a1f785_1841892e9df07039bbe4c3a55d11026a_finalresults';
204
205 % 'SVM 52dim feat, ID-sampling'
206 pca52 = 'runlog_7d5fafec0dc504215acc8cb7a9202a56_1841892e9df07039bbe4c3a55d11026a_finalresults';
207 % ---
208 % NOTE: all 4 test-set runs have exactly the same result
209 % ---
210 run1 = -1;
211 run2 = -1;
212 mode = 'join_datasets'; % '', 'avgbase', 'join_datasets'
213
214 'AllAvg vs Pca12'
215 [p, med, avg] = test_generic_significance_signrank(allavg,run1,pca12,run2,0,mode);
216 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
217
218 'AllAvg vs pca52'
219 [p, med, avg] = test_generic_significance_signrank(allavg,run1,pca52,run2,0,mode);
220 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
221
222 'Pca12 vs pca52'
223 [p, med, avg] = test_generic_significance_signrank(pca12,run1,pca52,run2,0,mode);
224 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
225
226
227 %%
228 % ------------------------------ Algorithms Weighted Training Weighted Performance Compared
229 'MLR t:w,e:w vs euclidean t:w,e:w'
230 % ---
231 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults';
232
233 % euclidean baseline, file same as the unweighted above
234 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
235 % ---
236 % NOTE: using the "weighted" parameter here
237 % ---
238 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode);
239 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
240
241 %%
242 'DMLR t:w,e:w vs euclidean t:w,e:w'
243 % ---
244 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults';
245
246 % euclidean baseline, file same as the unweighted above
247 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
248 % ---
249 % NOTE: using the "weighted" parameter here
250 % ---
251 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode);
252 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
253
254 %%
255 'SVM t:w,e:w vs euclidean t:w,e:w'
256 % ---
257 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults';
258
259 % euclidean baseline, file same as the unweighted above
260 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
261 % ---
262 % NOTE: using the "weighted" parameter here
263 % ---
264 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode);
265 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
266
267 %%
268 % ------------------------------ Algorithms Weighted Training UNWEIGHTED Performance Compared
269 'MLR t:w,e:uw vs euclidean t:w,e:uw'
270 % ---
271 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults';
272
273 % euclidean baseline, file same as the unweighted above
274 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
275 % ---
276 % NOTE: using the "weighted" parameter here
277 % ---
278 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 0 ,mode);
279 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
280
281 %%
282 'DMLR t:w,e:uw vs euclidean t:w,e:uw'
283 % ---
284 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults';
285
286 % euclidean baseline, file same as the unweighted above
287 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
288 % ---
289 % NOTE: using the "weighted" parameter here
290 % ---
291 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 0 ,mode);
292 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
293
294 %%
295 'SVM t:w,e:uw vs euclidean t:w,e:uw'
296 % ---
297 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults';
298
299 % euclidean baseline, file same as the unweighted above
300 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
301 % ---
302 % NOTE: using the "weighted" parameter here
303 % ---
304 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 0 ,mode);
305 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
306
307
308 %%
309 % ------------------------------ Algorithms w / uw Training WEIGHTED Performance Compared
310 'MLR t:w,e:w vs MLR t:uw,e:w'
311 % ---
312 % MLR t:w,e:w
313 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults';
314
315 % MLR t:uw,e:w
316 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a58077f1232c33b787b661039df107d_finalresults';
317 % ---
318 % NOTE: using the "weighted" parameter here
319 % ---
320 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode);
321 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
322
323 %%
324 % ---
325 'SVM t:w,e:w vs SVM t:uw,e:w'
326 % ---
327 % SVM t:w,e:w
328 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults';
329
330 % SVM t:uw,e:w
331 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults';
332 % ---
333 % NOTE: using the "weighted" parameter here
334 % ---
335 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode);
336 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
337
338 %%
339 % ------------------------------ Algorithms w / uw Training WEIGHTED Performance Compared
340 'DMLR t:w,e:w vs DMLR t:uw,e:w'
341 % ---
342 % DMLR t:w,e:w
343 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults';
344
345 % DMLR t:uw,e:w
346 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_bf08b16f453683d96ddffc31c6439730_finalresults';
347 % ---
348 % NOTE: using the "weighted" parameter here
349 % ---
350 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode);
351 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);