Mercurial > hg > camir-aes2014
comparison core/magnatagatune/makro_cupaper12_get_significance.m @ 0:e9a9cd732c1e tip
first hg version after svn
author | wolffd |
---|---|
date | Tue, 10 Feb 2015 15:05:51 +0000 |
parents | |
children |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:e9a9cd732c1e |
---|---|
1 % makro_get_significance_cupaper | |
2 %% | |
3 % ------------------------------ Algorithms Compared id-sampling | |
4 % --- | |
5 'mlr vs euclidean, mlr all average feat, ID-sampling' | |
6 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1a58077f1232c33b787b661039df107d_finalresults'; | |
7 | |
8 % euclidean average feat - TODO: which sampling is this ? | |
9 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
10 % --- | |
11 % NOTE: all 4 test-set runs have exactly the same result | |
12 % --- | |
13 run1 = -1; | |
14 run2 = -1; | |
15 mode = 'join_datasets'; % '', 'avgbase', 'join_datasets' | |
16 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run1,0,mode); | |
17 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
18 | |
19 | |
20 %% | |
21 % --- | |
22 'SVM vs euclidean, SVM all average feat, ID-sampling' | |
23 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults'; | |
24 | |
25 % euclidean average feat - TODO: which sampling is this ? | |
26 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
27 | |
28 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode); | |
29 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
30 | |
31 %% | |
32 % ------------------------------ Algorithms Compared TD-sampling | |
33 % --- | |
34 'mlr unclustered vs euclidean unclustered, mlr all average feat TD-sampling' | |
35 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_c364cb0803822d55f2940656c44b184d_finalresults'; | |
36 | |
37 % euclidean average feat TD-sampling | |
38 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_72303bfa642aad872665dee7a3b1e28c_finalresults'; | |
39 | |
40 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode); | |
41 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
42 | |
43 %% | |
44 % --- | |
45 'SVM unclustered vs euclidean unclustered, SVM all average feat TD-sampling' | |
46 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3434f0534fa910b26bbf927c65a7fb74_finalresults'; | |
47 | |
48 % euclidean average feat TD-sampling | |
49 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_72303bfa642aad872665dee7a3b1e28c_finalresults'; | |
50 % --- | |
51 | |
52 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode); | |
53 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
54 | |
55 | |
56 %% | |
57 % ------------------------------ Features Compared | |
58 'All 12-dim PCA Features compared' | |
59 % --- | |
60 namePCA12 = {'AcousticPCA12', ... | |
61 'Slaney08PCA12', ... | |
62 'TimbrePCA12', ... | |
63 'ChromaPCA12', ... | |
64 'AllPCA12', ... | |
65 'GenrePCA12'}; | |
66 % this is the index how the feature types appear in the paper | |
67 paperidx = [4 3 2 6 1 5]; | |
68 | |
69 filePCA12 = {'runlog_a18bd2111694ac59c9ba0a6810121796_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
70 'runlog_37e47c187886f73ec9a7d8dc24a84a52_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
71 'runlog_4c6787b403a07f5faf1ec26e891da4fa_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
72 'runlog_c5566f74e6a0d00b50f5eea05fdacfee_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
73 'runlog_e2c22696e7af9e7eea1fa1fd10a1f785_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
74 'runlog_efc6e5e9c56291cd1744092a1c59a293_1841892e9df07039bbe4c3a55d11026a_finalresults'}; | |
75 | |
76 | |
77 namePCA12 = namePCA12(paperidx); | |
78 filePCA12 = filePCA12(paperidx); | |
79 | |
80 p = zeros(numel(namePCA12),numel(namePCA12)); | |
81 med = zeros(numel(namePCA12),numel(namePCA12)); | |
82 avg = zeros(numel(namePCA12),numel(namePCA12)); | |
83 for i=1:numel(name) | |
84 for j = 1:i-1 | |
85 [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(filePCA12{i},-1,filePCA12{j},-1,0,mode); | |
86 end | |
87 % p(i,i) = 0.5; | |
88 end | |
89 % p = p + p'; | |
90 imagesc(p); | |
91 colormap(hot) | |
92 axis xy | |
93 set(gca,'XTick',1:numel(namePCA12), 'XTickLabel', namePCA12); | |
94 set(gca,'YTick',1:numel(namePCA12), 'YTickLabel', namePCA12); | |
95 matrix2latex(p,'%1.3f') | |
96 | |
97 %% | |
98 'All 52-dim PCA Features compared' | |
99 % --- | |
100 namePCA52 = {'GenrePCA52', ... | |
101 'AllPCA52', ... | |
102 'TimbrePCA52', ... | |
103 'AcousticPCA52', ... | |
104 'ChromaPCA52' ... | |
105 }; | |
106 % this is the index how the feature types appear in the paper | |
107 paperidx = [5 3 1 4 2]; | |
108 | |
109 filePCA52 = {'runlog_3cbf4759cf58af0728aaab0b5f2660e3_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
110 'runlog_7d5fafec0dc504215acc8cb7a9202a56_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
111 'runlog_a3c2c0a5742a42fd54497e69b8f44e8d_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
112 'runlog_c7164074206998aa184538bedcfdcf2f_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
113 'runlog_efbf7c8e75ae154c2f192acd08fbdcbc_1841892e9df07039bbe4c3a55d11026a_finalresults' ... | |
114 }; | |
115 | |
116 | |
117 namePCA52 = namePCA52(paperidx); | |
118 filePCA52 = filePCA52(paperidx); | |
119 | |
120 p = zeros(numel(namePCA52),numel(namePCA52)); | |
121 med = zeros(numel(namePCA52),numel(namePCA52)); | |
122 avg = zeros(numel(namePCA52),numel(namePCA52)); | |
123 for i=1:numel(name) | |
124 for j = 1:i-1 | |
125 [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(... | |
126 filePCA52{i},run1,filePCA52{j},run2,0,mode); | |
127 end | |
128 % p(i,i) = 0.5; | |
129 end | |
130 % p = p + p'; | |
131 imagesc(p); | |
132 colormap(hot) | |
133 axis xy | |
134 set(gca,'XTick',1:numel(namePCA52), 'XTickLabel', namePCA52); | |
135 set(gca,'YTick',1:numel(namePCA52), 'YTickLabel', namePCA52); | |
136 matrix2latex(p,'%1.3f') | |
137 | |
138 %% | |
139 'All full-dim Features compared (SVM)' | |
140 % --- | |
141 name = {'4Chroma', ... | |
142 '1Chroma', ... | |
143 '4Timbre', ... | |
144 '1Timbre', ... | |
145 '1Acoustic', ... | |
146 'Genre', ... | |
147 'Slaney08', ... | |
148 '1All', ... | |
149 '4All', ... | |
150 '4Acoustic' ... | |
151 }; | |
152 | |
153 % this is the index how the feature types appear in the paper | |
154 % Features & Chroma(1/4) & Timbre(1/4) & Slaney08 & Genre & Comb. Audio (1/4) & Comb. All(1/4) \\ | |
155 paperidx = [2 1 4 3 7 6 5 10 8 9]; | |
156 | |
157 file = {'runlog_20a2f6a0f20f488e9386ebb8c5026fcf_85c439d9ef3d135936e7645ebc0efe36_finalresults', ... | |
158 'runlog_20a2f6a0f20f488e9386ebb8c5026fcf_85c439d9ef3d135936e7645ebc0efe36_finalresults', ... | |
159 'runlog_3154f36c34c18f60218c5d3f0c0b5931_85c439d9ef3d135936e7645ebc0efe36_finalresults', ... | |
160 'runlog_3154f36c34c18f60218c5d3f0c0b5931_85c439d9ef3d135936e7645ebc0efe36_finalresults', ... | |
161 'runlog_31981d48dd0d25564ef3c2b3ca650b3b_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
162 'runlog_37867d3b5bd4c74b7b548732b80fb947_85c439d9ef3d135936e7645ebc0efe36_finalresults', ... | |
163 'runlog_f52d37439805ac4edc70b0432281abc3_85c439d9ef3d135936e7645ebc0efe36_finalresults', ... | |
164 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults', ... | |
165 'runlog_800d97be9ef6274dc3bbe6b9be2406a6_1a58077f1232c33b787b661039df107d_finalresults', ... | |
166 'runlog_cf5a61cca09e2a3182b794b70ee1ab91_1841892e9df07039bbe4c3a55d11026a_finalresults' ... | |
167 }; | |
168 | |
169 sets2join = {[1:4],[5:8],[1:4],[5:8],[],[],[],[],[],[]}; | |
170 | |
171 name = name(paperidx); | |
172 file = file(paperidx); | |
173 | |
174 p = zeros(numel(name),numel(name)); | |
175 med = zeros(numel(name),numel(name)); | |
176 avg = zeros(numel(name),numel(name)); | |
177 for i=1:numel(name) | |
178 for j = 1:i-1 | |
179 [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(... | |
180 file{i},sets2join{i},file{j},sets2join{j},0,mode); | |
181 end | |
182 % p(i,i) = 0.5; | |
183 end | |
184 p = flipud(p); | |
185 % p = p + p'; | |
186 imagesc(p); | |
187 colormap(hot) | |
188 axis ij | |
189 set(gca,'XTick',1:numel(name), 'XTickLabel', name); | |
190 set(gca,'YTick',1:numel(name), 'YTickLabel', fliplr(name)); | |
191 matrix2latex(p,'%1.3f') | |
192 | |
193 %% | |
194 'Comparing Feature dimensions and PCA effect on combined all features' | |
195 % --- | |
196 % 'SVM all average feat, ID-sampling' | |
197 allavg = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults'; | |
198 | |
199 % 'SVM 4 cluster feat, ID-sampling' | |
200 allfourcluster = 'runlog_800d97be9ef6274dc3bbe6b9be2406a6_1a58077f1232c33b787b661039df107d_finalresults'; | |
201 | |
202 % 'SVM 12dim feat, ID-sampling' | |
203 pca12 = 'runlog_e2c22696e7af9e7eea1fa1fd10a1f785_1841892e9df07039bbe4c3a55d11026a_finalresults'; | |
204 | |
205 % 'SVM 52dim feat, ID-sampling' | |
206 pca52 = 'runlog_7d5fafec0dc504215acc8cb7a9202a56_1841892e9df07039bbe4c3a55d11026a_finalresults'; | |
207 % --- | |
208 % NOTE: all 4 test-set runs have exactly the same result | |
209 % --- | |
210 run1 = -1; | |
211 run2 = -1; | |
212 mode = 'join_datasets'; % '', 'avgbase', 'join_datasets' | |
213 | |
214 'AllAvg vs Pca12' | |
215 [p, med, avg] = test_generic_significance_signrank(allavg,run1,pca12,run2,0,mode); | |
216 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
217 | |
218 'AllAvg vs pca52' | |
219 [p, med, avg] = test_generic_significance_signrank(allavg,run1,pca52,run2,0,mode); | |
220 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
221 | |
222 'Pca12 vs pca52' | |
223 [p, med, avg] = test_generic_significance_signrank(pca12,run1,pca52,run2,0,mode); | |
224 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
225 | |
226 | |
227 %% | |
228 % ------------------------------ Algorithms Weighted Training Weighted Performance Compared | |
229 'MLR t:w,e:w vs euclidean t:w,e:w' | |
230 % --- | |
231 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults'; | |
232 | |
233 % euclidean baseline, file same as the unweighted above | |
234 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
235 % --- | |
236 % NOTE: using the "weighted" parameter here | |
237 % --- | |
238 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode); | |
239 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
240 | |
241 %% | |
242 'DMLR t:w,e:w vs euclidean t:w,e:w' | |
243 % --- | |
244 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults'; | |
245 | |
246 % euclidean baseline, file same as the unweighted above | |
247 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
248 % --- | |
249 % NOTE: using the "weighted" parameter here | |
250 % --- | |
251 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode); | |
252 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
253 | |
254 %% | |
255 'SVM t:w,e:w vs euclidean t:w,e:w' | |
256 % --- | |
257 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults'; | |
258 | |
259 % euclidean baseline, file same as the unweighted above | |
260 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
261 % --- | |
262 % NOTE: using the "weighted" parameter here | |
263 % --- | |
264 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode); | |
265 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
266 | |
267 %% | |
268 % ------------------------------ Algorithms Weighted Training UNWEIGHTED Performance Compared | |
269 'MLR t:w,e:uw vs euclidean t:w,e:uw' | |
270 % --- | |
271 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults'; | |
272 | |
273 % euclidean baseline, file same as the unweighted above | |
274 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
275 % --- | |
276 % NOTE: using the "weighted" parameter here | |
277 % --- | |
278 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 0 ,mode); | |
279 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
280 | |
281 %% | |
282 'DMLR t:w,e:uw vs euclidean t:w,e:uw' | |
283 % --- | |
284 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults'; | |
285 | |
286 % euclidean baseline, file same as the unweighted above | |
287 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
288 % --- | |
289 % NOTE: using the "weighted" parameter here | |
290 % --- | |
291 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 0 ,mode); | |
292 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
293 | |
294 %% | |
295 'SVM t:w,e:uw vs euclidean t:w,e:uw' | |
296 % --- | |
297 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults'; | |
298 | |
299 % euclidean baseline, file same as the unweighted above | |
300 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults'; | |
301 % --- | |
302 % NOTE: using the "weighted" parameter here | |
303 % --- | |
304 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 0 ,mode); | |
305 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
306 | |
307 | |
308 %% | |
309 % ------------------------------ Algorithms w / uw Training WEIGHTED Performance Compared | |
310 'MLR t:w,e:w vs MLR t:uw,e:w' | |
311 % --- | |
312 % MLR t:w,e:w | |
313 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults'; | |
314 | |
315 % MLR t:uw,e:w | |
316 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a58077f1232c33b787b661039df107d_finalresults'; | |
317 % --- | |
318 % NOTE: using the "weighted" parameter here | |
319 % --- | |
320 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode); | |
321 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
322 | |
323 %% | |
324 % --- | |
325 'SVM t:w,e:w vs SVM t:uw,e:w' | |
326 % --- | |
327 % SVM t:w,e:w | |
328 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults'; | |
329 | |
330 % SVM t:uw,e:w | |
331 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults'; | |
332 % --- | |
333 % NOTE: using the "weighted" parameter here | |
334 % --- | |
335 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode); | |
336 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); | |
337 | |
338 %% | |
339 % ------------------------------ Algorithms w / uw Training WEIGHTED Performance Compared | |
340 'DMLR t:w,e:w vs DMLR t:uw,e:w' | |
341 % --- | |
342 % DMLR t:w,e:w | |
343 file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults'; | |
344 | |
345 % DMLR t:uw,e:w | |
346 file2 = 'runlog_0b506247a68167addf97fcb0296650eb_bf08b16f453683d96ddffc31c6439730_finalresults'; | |
347 % --- | |
348 % NOTE: using the "weighted" parameter here | |
349 % --- | |
350 [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2, 1 ,mode); | |
351 fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg); |