wolffd@0: function out = test_generic_features_parameters_crossval...
wolffd@0:     (fparams_all, trainparams_all, trainfun, featuretype)
wolffd@0: % 
wolffd@0: % this is a special function designed for the ISMIR 2012 
wolffd@0: % publication. A fixed and partitioned similarity dataset is used,
wolffd@0: % and the clip set as well as the features are SPECIFICALLY SELECTED
wolffd@0: % to each combination of training and test sets.
wolffd@0: 
wolffd@0: % svn hook   
wolffd@0: my_revision = str2double(substr('$Rev$',  5, -1));
wolffd@0: [ST,I] = dbstack();
wolffd@0: testscript = ST(end).name;
wolffd@0: 
wolffd@0: global globalvars;
wolffd@0: global comparison_ids;
wolffd@0: global comparison_invids;
wolffd@0: 
wolffd@0: eval(sprintf('global %s', MTTAudioFeatureDBgen.db_name(featuretype)));
wolffd@0: 
wolffd@0: % ---
wolffd@0: % get all combinations resulting from the above parameter set 
wolffd@0: % descriptions
wolffd@0: % ---
wolffd@0: fparams = param_combinations(fparams_all);
wolffd@0: 
wolffd@0: trainparams = param_combinations(trainparams_all);
wolffd@0: 
wolffd@0: % ---
wolffd@0: % the clips: sorted by comparison_id
wolffd@0: %
wolffd@0: % Now the clip type can be specified in the first training set
wolffd@0: % ---
wolffd@0: tmp_simdata = load(trainparams(1).dataset);
wolffd@0: if isfield(tmp_simdata, 'comparison_ids');
wolffd@0:     comparison_ids = tmp_simdata.comparison_ids;
wolffd@0:     %comparison_invids = tmp_simdata.comparison_invids;
wolffd@0:     clip_type = tmp_simdata.clip_type;
wolffd@0: else
wolffd@0:     load ('db.mat', 'comparison','comparison_ids','comparison_names');
wolffd@0:    
wolffd@0: 	clip_type = 'MTTClip';
wolffd@0: end
wolffd@0: comparison_invids = sparse(comparison_ids,1,[1:numel(comparison_ids)]);
wolffd@0: clips = feval(clip_type,comparison_ids);
wolffd@0: 
wolffd@0: % ---
wolffd@0: % provide some timing information
wolffd@0: % ---
wolffd@0: nruns = numel(fparams);
wolffd@0: tatic = [];
wolffd@0: tatoc = [];
wolffd@0: ftoc = [];
wolffd@0: mlrtic = [];
wolffd@0: mlrtoc = [];
wolffd@0: runs = 0;
wolffd@0: 
wolffd@0: res = [];
wolffd@0: % TEST
wolffd@0: for i = 1:numel(fparams)
wolffd@0: 
wolffd@0:     % TIMING start
wolffd@0:     runs = runs + 1;
wolffd@0:     tatic(end+1) = cputime();
wolffd@0:     
wolffd@0:     % warning ('disabled feature cache reset');
wolffd@0:     MTTAudioFeatureDBgen.reset_feature_dbs('exclude',{'db_magnaaudiofeat'});
wolffd@0:     eval('%s.reset;', MTTAudioFeatureDBgen.db_name(featuretype));
wolffd@0:     
wolffd@0:     % extract features 
wolffd@0:     try
wolffd@0:         % ---
wolffd@0:         % try loading old features with the same parameter hash.
wolffd@0:         % we use the md5 hash to distinguish between features
wolffd@0:         % ---
wolffd@0:         %paramhash = MTTAudioFeature.param_hash(featuretype, fparams(i));
wolffd@0:         %featfile = sprintf('runlog_%s_feat.mat', paramhash);
wolffd@0:         %if exist(featfile,'file') == 2
wolffd@0:         %   eval(sprintf('%s.import(featfile);', MTTAudioFeatureDBgen.db_name(featuretype)));
wolffd@0:         
wolffd@0:         % load db
wolffd@0:         paramhash=eval(sprintf('%s.load(featuretype, fparams(i), clips);', MTTAudioFeatureDBgen.db_name(featuretype)));
wolffd@0:         paramhash=substr(paramhash,  0, -4);
wolffd@0:         if ~isempty(paramhash)
wolffd@0:             % ---
wolffd@0:             % Here, we make sure the clips are 
wolffd@0:             % associated to the feature values
wolffd@0:             % ---
wolffd@0:             features = clips.features(featuretype, fparams(i));
wolffd@0:             X = features.vector();
wolffd@0:             
wolffd@0:         else
wolffd@0:             % ---
wolffd@0:             % we extract the mixed features, but leave the option of 
wolffd@0:             % using no tags
wolffd@0:             % ---
wolffd@0:             features = clips.features(featuretype, fparams(i));
wolffd@0:             features.define_global_transform();
wolffd@0:             
wolffd@0:             % get the feature vector
wolffd@0:             X = features.vector();
wolffd@0:             
wolffd@0: %             % save features with specific filename to disc
wolffd@0: %             xml_save(sprintf('runlog_%s_param.xml', paramhash), fparams(i));
wolffd@0: %             features.saveto(featfile);
wolffd@0: 
wolffd@0:             paramhash=eval(sprintf('%s.save();', MTTAudioFeatureDBgen.db_name(featuretype)));
wolffd@0:             paramhash=substr(paramhash,  0, -4);
wolffd@0:         end
wolffd@0:         
wolffd@0:         
wolffd@0:         % TIMING
wolffd@0:         ftoc(end+1) = cputime - tatic(end);
wolffd@0:         fprintf('Got features. took %2.2f minutes\n', ftoc(end) / 60);
wolffd@0:     catch err
wolffd@0:         
wolffd@0:         print_error(err);
wolffd@0:        
wolffd@0:         % ---
wolffd@0:         % TODO: save feature configuration and indicate faliure
wolffd@0:         % ---
wolffd@0:         if ~exist('paramhash')
wolffd@0:             paramhash = hash(xml_format(fparams(i)),'MD5');
wolffd@0:         end
wolffd@0:         xml_save(sprintf('runlog_%s_param.xml', paramhash), fparams(i));
wolffd@0:         xml_save(sprintf('runlog_%s_err.xml', paramhash), print_error(err));
wolffd@0:         
wolffd@0:         continue;
wolffd@0:     end
wolffd@0: 
wolffd@0:        
wolffd@0:     % skip empty feature set;
wolffd@0:     if isempty(X)
wolffd@0:         continue;
wolffd@0:     end
wolffd@0:     % iterate over trainparams
wolffd@0:     for j = 1:numel(trainparams)
wolffd@0:         
wolffd@0:         % TIMING
wolffd@0:         mlrtic(end+1) = cputime;
wolffd@0:         paramhash_mlr = hash(xml_format(trainparams(j)),'MD5');
wolffd@0:         
wolffd@0:         % ---
wolffd@0:         % Note: here we load the similarity data.
wolffd@0:         %       this data is trunated if inctrain
wolffd@0:         % ---
wolffd@0:         simdata = load(trainparams(j).dataset);
wolffd@0:         if isfield(trainparams(j),'inctrain') && (trainparams(j).inctrain == 0)
wolffd@0:             
wolffd@0:             simdata.partBinTrn = simdata.partBinTrn(:,end);
wolffd@0:             
wolffd@0:             if isfield(simdata, 'partBinNoTrn')
wolffd@0:                 simdata.partBinNoTrn = simdata.partBinNoTrn(:,end);
wolffd@0:             end
wolffd@0:         end
wolffd@0:         
wolffd@0:         if isfield(trainparams(j),'notintrain') && (trainparams(j).notintrain == 0)
wolffd@0:             
wolffd@0:             if isfield(simdata, 'partBinNoTrn')
wolffd@0:                 simdata = rmfield(simdata, 'partBinNoTrn');
wolffd@0:             end    
wolffd@0:         end
wolffd@0:         
wolffd@0:         % ---
wolffd@0:         % NOTE: THIS IS TRAINING
wolffd@0:         % call training function
wolffd@0:         % ---
wolffd@0:         [tmp] = do_test_rounds(trainfun, X, simdata, trainparams(j), fparams(i),...
wolffd@0:             paramhash, paramhash_mlr, clips);
wolffd@0:         
wolffd@0:         tmp.finfo = features(1).data.info;
wolffd@0:         tmp.fparams = features(1).my_params;
wolffd@0:         tmp.fparamhash = paramhash;
wolffd@0:         tmp.script = testscript;
wolffd@0:  
wolffd@0:         % TIMING 
wolffd@0:         mlrtoc(end+1) = cputime - mlrtic(end);
wolffd@0:         tmp.timeused = mlrtoc(end);
wolffd@0:         
wolffd@0:         % save result to result struct;
wolffd@0:         if ~isempty(tmp)
wolffd@0:             if isempty(res)
wolffd@0:                 
wolffd@0:                 res = tmp;
wolffd@0:             else
wolffd@0:                 
wolffd@0:                 res(end+1) = tmp;
wolffd@0:             end
wolffd@0:             
wolffd@0:             if size(tmp.mean_ok_test,1) == 2
wolffd@0:                 maxperf = max(max(tmp.mean_ok_test(1,:)));
wolffd@0:             else
wolffd@0:                 maxperf = max(max(tmp.mean_ok_test));
wolffd@0:             end
wolffd@0:             fprintf('Learned something: %2.2f perc. Took %2.2f minutes \n',...
wolffd@0:                  maxperf * 100, mlrtoc(end) / 60 );
wolffd@0:             pause(0.5);
wolffd@0:         else
wolffd@0:             warning('Learned nothing \n');
wolffd@0:         end
wolffd@0:         
wolffd@0:         % save output
wolffd@0:         out = res;
wolffd@0:         
wolffd@0:         % save final results
wolffd@0:         save(sprintf('runlog_%s_%s_finalresults.mat',...
wolffd@0:                 hash(xml_format(fparams),'MD5'),...
wolffd@0:                 hash(xml_format(trainparams),'MD5')...
wolffd@0:                 ), 'out');
wolffd@0: 
wolffd@0:         xml_save(sprintf('runlog_%s_%s_params.mat',...
wolffd@0:                 hash(xml_format(fparams),'MD5'),...
wolffd@0:                 hash(xml_format(trainparams),'MD5')...
wolffd@0:                 ), struct('fparams', fparams_all, 'trainparams', trainparams_all) );
wolffd@0: 
wolffd@0:     end
wolffd@0:     
wolffd@0:     % TIMING
wolffd@0:     clc;
wolffd@0:     tatoc(end+1) = cputime - tatic(end);
wolffd@0:     cprint(0,'%3.2f percent done, %2.2fh spent, %2.2fh to go. \n mlr / feature: %3.3f \n',...
wolffd@0:         (runs / nruns) *100 , sum(tatoc) / 3600,...
wolffd@0:         ((sum(tatoc) / runs) * (nruns - runs)) / 3600, mean(mlrtoc) / mean(ftoc) );
wolffd@0:      cprint(0,'\nGo get a coffee, the next round will take %3.0f minutes \n', ...
wolffd@0:                 (mean(mlrtoc) * numel(trainparams) + mean(ftoc)) / 60);
wolffd@0: end
wolffd@0: end
wolffd@0: 
wolffd@0: