wolffd@0: % makro_get_significance_cupaper
wolffd@0: %%
wolffd@0: % ------------------------------ Algorithms Compared id-sampling
wolffd@0: % ---
wolffd@0: 'mlr vs euclidean, mlr all average feat, ID-sampling'
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1a58077f1232c33b787b661039df107d_finalresults';
wolffd@0: 
wolffd@0: % euclidean average feat - TODO: which sampling is this ?
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: all 4 test-set runs have exactly the same result 
wolffd@0: % ---
wolffd@0: run1 = -1;
wolffd@0: run2 = -1;
wolffd@0: mode = 'join_datasets'; % '', 'avgbase', 'join_datasets'
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run1,0,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: 
wolffd@0: %%
wolffd@0: % ---
wolffd@0: 'SVM vs euclidean, SVM all average feat, ID-sampling'
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults';
wolffd@0: 
wolffd@0: % euclidean average feat - TODO: which sampling is this ?
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: 
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: % ------------------------------ Algorithms Compared TD-sampling
wolffd@0: % ---
wolffd@0: 'mlr unclustered vs euclidean unclustered, mlr all average feat TD-sampling'
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_c364cb0803822d55f2940656c44b184d_finalresults';
wolffd@0: 
wolffd@0: % euclidean average feat TD-sampling
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_72303bfa642aad872665dee7a3b1e28c_finalresults';
wolffd@0: 
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: % ---
wolffd@0: 'SVM unclustered vs euclidean unclustered, SVM all average feat TD-sampling'
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3434f0534fa910b26bbf927c65a7fb74_finalresults';
wolffd@0: 
wolffd@0: % euclidean average feat TD-sampling
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_72303bfa642aad872665dee7a3b1e28c_finalresults';
wolffd@0: % ---
wolffd@0: 
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,0,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: 
wolffd@0: %%
wolffd@0: % ------------------------------ Features Compared 
wolffd@0: 'All 12-dim PCA Features compared'
wolffd@0: % ---
wolffd@0: namePCA12 = {'AcousticPCA12', ...
wolffd@0:     'Slaney08PCA12', ...
wolffd@0:     'TimbrePCA12', ...
wolffd@0:     'ChromaPCA12', ...
wolffd@0:     'AllPCA12', ...
wolffd@0:     'GenrePCA12'};
wolffd@0: % this is the index how the feature types appear in the paper
wolffd@0: paperidx = [4 3 2 6 1 5];
wolffd@0: 
wolffd@0: filePCA12 = {'runlog_a18bd2111694ac59c9ba0a6810121796_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_37e47c187886f73ec9a7d8dc24a84a52_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_4c6787b403a07f5faf1ec26e891da4fa_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_c5566f74e6a0d00b50f5eea05fdacfee_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_e2c22696e7af9e7eea1fa1fd10a1f785_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_efc6e5e9c56291cd1744092a1c59a293_1841892e9df07039bbe4c3a55d11026a_finalresults'};
wolffd@0: 
wolffd@0: 
wolffd@0: namePCA12 = namePCA12(paperidx);
wolffd@0: filePCA12 = filePCA12(paperidx);
wolffd@0: 
wolffd@0: p = zeros(numel(namePCA12),numel(namePCA12));
wolffd@0: med = zeros(numel(namePCA12),numel(namePCA12));
wolffd@0: avg = zeros(numel(namePCA12),numel(namePCA12));
wolffd@0: for i=1:numel(name)
wolffd@0:     for j = 1:i-1
wolffd@0:         [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(filePCA12{i},-1,filePCA12{j},-1,0,mode);
wolffd@0:     end
wolffd@0:    % p(i,i) = 0.5;
wolffd@0: end
wolffd@0: % p = p + p';
wolffd@0: imagesc(p);
wolffd@0: colormap(hot)
wolffd@0: axis xy
wolffd@0: set(gca,'XTick',1:numel(namePCA12), 'XTickLabel', namePCA12);
wolffd@0: set(gca,'YTick',1:numel(namePCA12), 'YTickLabel', namePCA12);
wolffd@0: matrix2latex(p,'%1.3f')
wolffd@0: 
wolffd@0: %%
wolffd@0: 'All 52-dim PCA Features compared'
wolffd@0: % ---
wolffd@0: namePCA52 = {'GenrePCA52', ...
wolffd@0:     'AllPCA52', ...
wolffd@0:     'TimbrePCA52', ...
wolffd@0:     'AcousticPCA52', ...
wolffd@0:     'ChromaPCA52' ...
wolffd@0:      };
wolffd@0: % this is the index how the feature types appear in the paper
wolffd@0: paperidx = [5 3 1 4 2];
wolffd@0: 
wolffd@0: filePCA52 = {'runlog_3cbf4759cf58af0728aaab0b5f2660e3_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_7d5fafec0dc504215acc8cb7a9202a56_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_a3c2c0a5742a42fd54497e69b8f44e8d_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_c7164074206998aa184538bedcfdcf2f_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_efbf7c8e75ae154c2f192acd08fbdcbc_1841892e9df07039bbe4c3a55d11026a_finalresults' ...
wolffd@0:     };
wolffd@0: 
wolffd@0: 
wolffd@0: namePCA52 = namePCA52(paperidx);
wolffd@0: filePCA52 = filePCA52(paperidx);
wolffd@0: 
wolffd@0: p = zeros(numel(namePCA52),numel(namePCA52));
wolffd@0: med = zeros(numel(namePCA52),numel(namePCA52));
wolffd@0: avg = zeros(numel(namePCA52),numel(namePCA52));
wolffd@0: for i=1:numel(name)
wolffd@0:     for j = 1:i-1
wolffd@0:         [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(...
wolffd@0:             filePCA52{i},run1,filePCA52{j},run2,0,mode);
wolffd@0:     end
wolffd@0:    % p(i,i) = 0.5;
wolffd@0: end
wolffd@0: % p = p + p';
wolffd@0: imagesc(p);
wolffd@0: colormap(hot)
wolffd@0: axis xy
wolffd@0: set(gca,'XTick',1:numel(namePCA52), 'XTickLabel', namePCA52);
wolffd@0: set(gca,'YTick',1:numel(namePCA52), 'YTickLabel', namePCA52);
wolffd@0: matrix2latex(p,'%1.3f')
wolffd@0: 
wolffd@0: %%
wolffd@0: 'All full-dim Features compared (SVM)'
wolffd@0: % ---
wolffd@0: name = {'4Chroma', ...
wolffd@0:     '1Chroma', ...
wolffd@0:     '4Timbre', ...
wolffd@0:     '1Timbre', ...
wolffd@0:     '1Acoustic', ...
wolffd@0:     'Genre', ...
wolffd@0:     'Slaney08', ...
wolffd@0:     '1All', ...
wolffd@0:     '4All', ...
wolffd@0:     '4Acoustic' ...
wolffd@0:     };
wolffd@0: 
wolffd@0: % this is the index how the feature types appear in the paper
wolffd@0: % Features & Chroma(1/4) & Timbre(1/4) & Slaney08 & Genre & Comb. Audio (1/4) & Comb. All(1/4) \\
wolffd@0: paperidx = [2 1 4 3 7 6 5 10 8 9];
wolffd@0: 
wolffd@0: file = {'runlog_20a2f6a0f20f488e9386ebb8c5026fcf_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
wolffd@0:     'runlog_20a2f6a0f20f488e9386ebb8c5026fcf_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
wolffd@0:     'runlog_3154f36c34c18f60218c5d3f0c0b5931_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
wolffd@0:     'runlog_3154f36c34c18f60218c5d3f0c0b5931_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
wolffd@0:     'runlog_31981d48dd0d25564ef3c2b3ca650b3b_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_37867d3b5bd4c74b7b548732b80fb947_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
wolffd@0:     'runlog_f52d37439805ac4edc70b0432281abc3_85c439d9ef3d135936e7645ebc0efe36_finalresults', ...
wolffd@0:     'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults', ...
wolffd@0:     'runlog_800d97be9ef6274dc3bbe6b9be2406a6_1a58077f1232c33b787b661039df107d_finalresults', ...
wolffd@0:     'runlog_cf5a61cca09e2a3182b794b70ee1ab91_1841892e9df07039bbe4c3a55d11026a_finalresults' ...
wolffd@0:     };
wolffd@0: 
wolffd@0: sets2join = {[1:4],[5:8],[1:4],[5:8],[],[],[],[],[],[]};
wolffd@0: 
wolffd@0: name = name(paperidx);
wolffd@0: file = file(paperidx);
wolffd@0: 
wolffd@0: p = zeros(numel(name),numel(name));
wolffd@0: med = zeros(numel(name),numel(name));
wolffd@0: avg = zeros(numel(name),numel(name));
wolffd@0: for i=1:numel(name)
wolffd@0:     for j = 1:i-1
wolffd@0:         [p(i,j), med(i,j), avg(i,j)] = test_generic_significance_signrank(...
wolffd@0:             file{i},sets2join{i},file{j},sets2join{j},0,mode);
wolffd@0:     end
wolffd@0:    % p(i,i) = 0.5;
wolffd@0: end
wolffd@0: p = flipud(p);
wolffd@0: % p = p + p';
wolffd@0: imagesc(p);
wolffd@0: colormap(hot)
wolffd@0: axis ij
wolffd@0: set(gca,'XTick',1:numel(name), 'XTickLabel', name);
wolffd@0: set(gca,'YTick',1:numel(name), 'YTickLabel', fliplr(name));
wolffd@0: matrix2latex(p,'%1.3f')
wolffd@0: 
wolffd@0: %%
wolffd@0: 'Comparing Feature dimensions and PCA effect on combined all features'
wolffd@0: % ---
wolffd@0: % 'SVM all average feat, ID-sampling'
wolffd@0: allavg = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults';
wolffd@0: 
wolffd@0: % 'SVM 4 cluster feat, ID-sampling'
wolffd@0: allfourcluster = 'runlog_800d97be9ef6274dc3bbe6b9be2406a6_1a58077f1232c33b787b661039df107d_finalresults';
wolffd@0: 
wolffd@0: % 'SVM 12dim feat, ID-sampling'
wolffd@0: pca12 = 'runlog_e2c22696e7af9e7eea1fa1fd10a1f785_1841892e9df07039bbe4c3a55d11026a_finalresults';
wolffd@0: 
wolffd@0: % 'SVM 52dim feat, ID-sampling'
wolffd@0: pca52 = 'runlog_7d5fafec0dc504215acc8cb7a9202a56_1841892e9df07039bbe4c3a55d11026a_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: all 4 test-set runs have exactly the same result 
wolffd@0: % ---
wolffd@0: run1 = -1;
wolffd@0: run2 = -1;
wolffd@0: mode = 'join_datasets'; % '', 'avgbase', 'join_datasets'
wolffd@0: 
wolffd@0: 'AllAvg vs Pca12'
wolffd@0: [p, med, avg] = test_generic_significance_signrank(allavg,run1,pca12,run2,0,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: 'AllAvg vs pca52'
wolffd@0: [p, med, avg] = test_generic_significance_signrank(allavg,run1,pca52,run2,0,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: 'Pca12 vs pca52'
wolffd@0: [p, med, avg] = test_generic_significance_signrank(pca12,run1,pca52,run2,0,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: 
wolffd@0: %%
wolffd@0: % ------------------------------ Algorithms Weighted Training Weighted Performance Compared 
wolffd@0: 'MLR t:w,e:w vs euclidean t:w,e:w'
wolffd@0: % ---
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults';
wolffd@0: 
wolffd@0: % euclidean baseline, file same as the unweighted above
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  1   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: 'DMLR t:w,e:w vs euclidean t:w,e:w'
wolffd@0: % ---
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults';
wolffd@0: 
wolffd@0: % euclidean baseline, file same as the unweighted above
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  1   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: 'SVM t:w,e:w vs euclidean t:w,e:w'
wolffd@0: % ---
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults';
wolffd@0: 
wolffd@0: % euclidean baseline, file same as the unweighted above
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  1   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: % ------------------------------ Algorithms Weighted Training UNWEIGHTED Performance Compared 
wolffd@0: 'MLR t:w,e:uw vs euclidean t:w,e:uw'
wolffd@0: % ---
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults';
wolffd@0: 
wolffd@0: % euclidean baseline, file same as the unweighted above
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  0   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: 'DMLR t:w,e:uw vs euclidean t:w,e:uw'
wolffd@0: % ---
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults';
wolffd@0: 
wolffd@0: % euclidean baseline, file same as the unweighted above
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  0   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: 'SVM t:w,e:uw vs euclidean t:w,e:uw'
wolffd@0: % ---
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults';
wolffd@0: 
wolffd@0: % euclidean baseline, file same as the unweighted above
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a9788c5e14f30e23ed2a05dbf513c9f_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  0   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: 
wolffd@0: %%
wolffd@0: % ------------------------------ Algorithms w / uw Training WEIGHTED Performance Compared 
wolffd@0: 'MLR t:w,e:w vs MLR t:uw,e:w'
wolffd@0: % ---
wolffd@0: % MLR t:w,e:w
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_3cdcca7596fed97f87b0ec051cb8bba0_finalresults';
wolffd@0: 
wolffd@0: % MLR t:uw,e:w
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1a58077f1232c33b787b661039df107d_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  1   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: % ---
wolffd@0: 'SVM t:w,e:w vs SVM t:uw,e:w'
wolffd@0: % ---
wolffd@0: % SVM t:w,e:w
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_9bd9ccddb2e4e622e2ba9826466442ba_finalresults';
wolffd@0: 
wolffd@0: % SVM t:uw,e:w
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_1841892e9df07039bbe4c3a55d11026a_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  1   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);
wolffd@0: 
wolffd@0: %%
wolffd@0: % ------------------------------ Algorithms w / uw Training WEIGHTED Performance Compared 
wolffd@0: 'DMLR t:w,e:w vs DMLR t:uw,e:w'
wolffd@0: % ---
wolffd@0: % DMLR t:w,e:w
wolffd@0: file1 = 'runlog_0b506247a68167addf97fcb0296650eb_1cc76d534804229cbdec8b20f8b75dba_finalresults';
wolffd@0: 
wolffd@0: % DMLR t:uw,e:w
wolffd@0: file2 = 'runlog_0b506247a68167addf97fcb0296650eb_bf08b16f453683d96ddffc31c6439730_finalresults';
wolffd@0: % ---
wolffd@0: % NOTE: using the "weighted" parameter here
wolffd@0: % ---
wolffd@0: [p, med, avg] = test_generic_significance_signrank(file1,run1,file2,run2,  1   ,mode);
wolffd@0: fprintf('p = %3.4f, median = %3.4f, avg = %3.4f \n\n',p,med,avg);