wolffd@0: classdef MTTAudioFeatureBasicSm < MTTAudioFeature & handle wolffd@0: % --- wolffd@0: % the MTTAudioFeatureBasicSm Class contains wolffd@0: % a basic summary of chroma, mfcc and tempo features wolffd@0: % a few common chroma and mfcc vectors are concatenated wolffd@0: % along with some clip-wide variance wolffd@0: % a metric / rhythm fingerprint is added wolffd@0: % wolffd@0: % The usual workflow for these features consists of three steps wolffd@0: % 1. extract: extracts the basic single-file dependent features wolffd@0: % 2. define_global_transform: calculates the global feature wolffd@0: % transformation parameters wolffd@0: % 3. finalise: applies the common transformations to a specific feature wolffd@0: % --- wolffd@0: wolffd@0: properties(Constant = true) wolffd@0: wolffd@0: % svn hook wolffd@0: my_revision = str2double(substr('$Rev$', 5, -1)); wolffd@0: end wolffd@0: wolffd@0: properties wolffd@0: % --- wolffd@0: % Set default parameters wolffd@0: % --- wolffd@0: my_params = struct(... wolffd@0: 'nchromas', 4, ... % 4 chroma vectors wolffd@0: 'chroma_var', 0, ... % chroma variance wolffd@0: 'norm_chromas', 0, ... % not implemented, chromas already rel. wolffd@0: 'min_kshift_chromas', 0.1, ... % treshold for key shift. set to 1 for no shift (0-1) wolffd@0: ... wolffd@0: 'ntimbres', 4, ... wolffd@0: 'timbre_var', 0, ... % timbre variance wolffd@0: 'norm_timbres', 1, ... wolffd@0: 'clip_timbres', 0.85, ... % percentile of data which has to be inside 0-1 bounds wolffd@0: ... wolffd@0: 'norm_weights',0, ... % globally norm weights for chroma times? wolffd@0: 'norm_interval',1, ... wolffd@0: 'max_iter',100, ... % max iterations for chroma and timbre knn wolffd@0: ... wolffd@0: 'nrhythms', 0, ... wolffd@0: 'nints', 11, ... wolffd@0: 'energy_sr', 1000, ... % sample rate for energy curve wolffd@0: 'norm_acorr', 1 ... % normalise arcorr locally-> shape imp... energy is normalised anyways wolffd@0: ); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % member functions wolffd@0: % --- wolffd@0: methods wolffd@0: wolffd@0: % --- wolffd@0: % constructor: pointer to feature in database wolffd@0: % --- wolffd@0: function feature = MTTAudioFeatureBasicSm(varargin) wolffd@0: wolffd@0: feature = feature@MTTAudioFeature(varargin{:}); wolffd@0: wolffd@0: end wolffd@0: % --- wolffd@0: % extract feature data from raw audio features wolffd@0: % --- wolffd@0: function data = extract(feature, clip) wolffd@0: % --- wolffd@0: % get Basic Summary audio features. this includes possible wolffd@0: % local normalisations wolffd@0: % --- wolffd@0: wolffd@0: global globalvars; wolffd@0: wolffd@0: % --- wolffd@0: % get casimir child clip if available wolffd@0: % --- wolffd@0: if isa(clip, 'CASIMIRClip') wolffd@0: baseclip = clip.child_clip(); wolffd@0: else wolffd@0: baseclip = clip; wolffd@0: end wolffd@0: if isa(baseclip, 'MTTClip') wolffd@0: rawf = baseclip.audio_features_raw(); wolffd@0: elseif isa(baseclip, 'MSDClip') wolffd@0: rawf = baseclip.features('MSDAudioFeatureRAW'); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % now extract the features wolffd@0: % first step: chroma clustering wolffd@0: % --- wolffd@0: weights = [rawf.data.segments_duration]; wolffd@0: wolffd@0: % normalise weights wolffd@0: weights = weights / rawf.data.duration; wolffd@0: wolffd@0: chroma = [rawf.data.segments_pitches]'; wolffd@0: wolffd@0: % --- wolffd@0: % get most present chroma vectors. wolffd@0: % the weighted k-means should return the four most prominent wolffd@0: % chroma vectors and their weight wolffd@0: % --- wolffd@0: % display error values wolffd@0: wolffd@0: op = foptions(); wolffd@0: op(1) = 0; wolffd@0: op(14) = feature.my_params.max_iter; wolffd@0: wolffd@0: % check for trivial case wolffd@0: if feature.my_params.nchromas == 0 wolffd@0: wolffd@0: chromas = []; wolffd@0: cwght = []; wolffd@0: wolffd@0: elseif feature.my_params.nchromas == 1 wolffd@0: wolffd@0: chromas = mean(chroma, 1); wolffd@0: chroma_var = var(chroma, 0, 1); wolffd@0: cwght = 1; wolffd@0: wolffd@0: elseif numel(weights) > feature.my_params.nchromas wolffd@0: wolffd@0: % --- wolffd@0: % there may be few chromas, try kmeans several (20) times wolffd@0: % --- wolffd@0: cont = 0; wolffd@0: cwght = []; wolffd@0: while (numel(cwght) ~= feature.my_params.nchromas) && (cont < 20); wolffd@0: wolffd@0: [chromas, cwght, post] = ... wolffd@0: weighted_kmeans(feature.my_params.nchromas, chroma, weights, op); wolffd@0: wolffd@0: cont = cont + 1; wolffd@0: end wolffd@0: wolffd@0: if (numel(cwght) ~= feature.my_params.nchromas) wolffd@0: wolffd@0: error('cannot find enough chroma centres'); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % Calculate the weighted variance of the chroma clusters wolffd@0: % --- wolffd@0: if feature.my_params.chroma_var >= 1 wolffd@0: wolffd@0: chroma_var = zeros(size(chromas)); wolffd@0: for i = 1:size(chroma_var,1) wolffd@0: wolffd@0: % get distance from cluster centroid wolffd@0: tmp_var = (chroma(post(:,i),:) - repmat(chromas(i,:), sum(post(:,i)),1)).^2; wolffd@0: wolffd@0: % add up the weighted differences and normalise by sum wolffd@0: % of weights wolffd@0: chroma_var(i,:) = (weights(post(:,i)) * tmp_var) ./... wolffd@0: (sum(weights(post(:,i)))); wolffd@0: end wolffd@0: end wolffd@0: else wolffd@0: % --- wolffd@0: % odd case: less than nchroma data points. wolffd@0: % we repeat the mean vector at the end wolffd@0: % --- wolffd@0: chromas = [chroma; repmat(mean(chroma, 1),... wolffd@0: feature.my_params.nchromas - numel(weights), 1 )]; wolffd@0: wolffd@0: cwght = weights; wolffd@0: cwght( end + 1:feature.my_params.nchromas ) = 0; wolffd@0: wolffd@0: % --- wolffd@0: % TODO: get a variance for odd case : wolffd@0: % replicate the complete data variance? wolffd@0: % NO: every vector is a clsuter => zero variance wolffd@0: % --- wolffd@0: end wolffd@0: wolffd@0: % trivial case: no variance requested wolffd@0: if ~exist('chroma_var','var') wolffd@0: chroma_var = zeros(size(chromas)); wolffd@0: end wolffd@0: wolffd@0: % sort by associated time wolffd@0: [cwght, idx] = sort(cwght, 'descend'); wolffd@0: chromas = chromas(idx,:); wolffd@0: chroma_var = chroma_var(idx,:); wolffd@0: wolffd@0: % --- wolffd@0: % shift according to detected key, but only if wolffd@0: % the confidencee is high enough wolffd@0: % --- wolffd@0: shift = 0; wolffd@0: if rawf.data.keyConfidence > feature.my_params.min_kshift_chromas; wolffd@0: wolffd@0: shift = -rawf.data.key; wolffd@0: chromas = circshift(chromas, [0 shift]); wolffd@0: chroma_var = circshift(chroma_var, [0 shift]); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % get mfcc centres: wolffd@0: % the same for mfccs wolffd@0: % --- wolffd@0: mfcc = [rawf.data.segments_timbre]'; wolffd@0: if feature.my_params.ntimbres == 0 wolffd@0: wolffd@0: mfccs = []; wolffd@0: mwght = []; wolffd@0: wolffd@0: elseif feature.my_params.ntimbres == 1 wolffd@0: wolffd@0: mfccs = mean(mfcc, 1); wolffd@0: timbre_var = var(mfccs, 0, 1); wolffd@0: mwght = 1; wolffd@0: wolffd@0: elseif numel(weights) > feature.my_params.ntimbres wolffd@0: wolffd@0: % --- wolffd@0: % there may be few mfccs, try kmeans several times wolffd@0: % --- wolffd@0: cont = 0; wolffd@0: mwght = []; wolffd@0: while (numel(mwght) ~= feature.my_params.ntimbres) && (cont < 20); wolffd@0: wolffd@0: [mfccs, mwght, post] = ... wolffd@0: weighted_kmeans(feature.my_params.ntimbres, mfcc, weights, op); wolffd@0: cont = cont + 1; wolffd@0: end wolffd@0: wolffd@0: if (numel(mwght) ~= feature.my_params.ntimbres) wolffd@0: wolffd@0: error('cannot find enough mfcc centres'); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % Calculate the weighted variance of the chroma clusters wolffd@0: % --- wolffd@0: if feature.my_params.timbre_var >= 1 wolffd@0: wolffd@0: timbre_var = zeros(size(mfccs)); wolffd@0: for i = 1:size(timbre_var,1) wolffd@0: wolffd@0: % get distance from cluster centroid wolffd@0: tmp_var = (mfcc(post(:,i),:) - repmat(mfccs(i,:), sum(post(:,i)),1)).^2; wolffd@0: wolffd@0: % add up the weighted differences and normalise by sum wolffd@0: % of weights wolffd@0: timbre_var(i,:) = (weights(post(:,i)) * tmp_var) ./... wolffd@0: (sum(weights(post(:,i)))); wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: else wolffd@0: % --- wolffd@0: % odd case: less than nchroma data points. wolffd@0: % we repeat the mean vector at the end wolffd@0: % --- wolffd@0: mfccs = [mfcc; repmat(mean(mfcc, 1),... wolffd@0: feature.my_params.ntimbres - numel(weights), 1)]; wolffd@0: mwght = weights; wolffd@0: mwght( end + 1:feature.my_params.ntimbres) = 0; wolffd@0: end wolffd@0: wolffd@0: % trivial case: no variance requested wolffd@0: if ~exist('timbre_var','var') wolffd@0: timbre_var = zeros(size(mfccs)); wolffd@0: end wolffd@0: wolffd@0: % sort by associated time wolffd@0: [mwght, idx] = sort(mwght, 'descend'); wolffd@0: mfccs = mfccs(idx,:); wolffd@0: timbre_var = timbre_var(idx,:); wolffd@0: wolffd@0: % --- wolffd@0: % get beat features: wolffd@0: % the autocorrelation curve over n quarters of length wolffd@0: % wolffd@0: % alternative: how about using the n=8 quarters relative wolffd@0: % volumes from the start of a sure measure? wolffd@0: % --- wolffd@0: if feature.my_params.nrhythms >= 1 wolffd@0: bars = rawf.data.bars; wolffd@0: beats = rawf.data.beats; wolffd@0: tatums = rawf.data.tatums; wolffd@0: % --- wolffd@0: % NOTE: the beat and tatum markers seem to have an offset :( wolffd@0: % --- wolffd@0: offset = 0.118; %seconds wolffd@0: wolffd@0: [envelope, time] = energy_envelope(feature, clip); wolffd@0: wolffd@0: % we offset the energy curve wolffd@0: time = time + offset; wolffd@0: wolffd@0: % --- wolffd@0: % we try to start at the best beat confidence more wolffd@0: % than sixteen eights from the end wolffd@0: % --- wolffd@0: wolffd@0: if rawf.data.tempo > 0 wolffd@0: wolffd@0: eightl = 30 / rawf.data.tempo; wolffd@0: else wolffd@0: % --- wolffd@0: % odd case: no rhythm data. assume 100 bpm wolffd@0: % --- wolffd@0: wolffd@0: eightl = 0.3; wolffd@0: end wolffd@0: wolffd@0: if isempty(beats) wolffd@0: % --- wolffd@0: % odd case: no beats detected. -> use best tatum wolffd@0: % --- wolffd@0: if ~isempty(tatums) wolffd@0: wolffd@0: beats = tatums; wolffd@0: else wolffd@0: wolffd@0: % ok, just take the beginning wolffd@0: beats = [0; 1]; wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: last_valid = find(beats(1,:) < ... wolffd@0: (rawf.data.duration - feature.my_params.nints * eightl),1, 'last'); wolffd@0: wolffd@0: % find the best valid beat postition wolffd@0: [null, max_measure] = max( beats(2, 1:last_valid)); wolffd@0: max_mtime = beats(1,max_measure); wolffd@0: wolffd@0: % --- wolffd@0: % the correlation is calculated for the estimated eights lenght wolffd@0: % and for the 16th intervals, respectively. wolffd@0: % --- wolffd@0: wolffd@0: % calculate the EIGHTS correlation for the following segment wolffd@0: [acorr8, eight_en, eightt] = ... wolffd@0: beat_histogram(feature, max_mtime, eightl, envelope, time); wolffd@0: wolffd@0: % calculate the SIXTEENTHS correlation for the following segment wolffd@0: [acorr16, six_en, sixt] = ... wolffd@0: beat_histogram(feature, max_mtime, eightl / 2, envelope, time); wolffd@0: wolffd@0: % --- wolffd@0: % save the various features wolffd@0: % --- wolffd@0: % save rythm feature data wolffd@0: wolffd@0: data.rhythm.acorr8 = acorr8; wolffd@0: data.rhythm.acorr8_lag = eightt(1:end/2)-eightt(1); wolffd@0: wolffd@0: data.rhythm.energy8 = eight_en(1:end/2); wolffd@0: data.rhythm.energy8_time = eightt(1:end/2); wolffd@0: wolffd@0: % -- wolffd@0: % the interval is normed locally up to a max value wolffd@0: % associated to 30bpm wolffd@0: % --- wolffd@0: if feature.my_params.norm_interval wolffd@0: wolffd@0: % 1 second max value wolffd@0: data.rhythm.interval8 = eightl / 2; wolffd@0: else wolffd@0: data.rhythm.interval8 = eightl / 2; wolffd@0: end wolffd@0: wolffd@0: if feature.my_params.nrhythms >= 2 wolffd@0: wolffd@0: data.rhythm.acorr16 = acorr16; wolffd@0: data.rhythm.acorr16_lag = data.rhythm.acorr8_lag / 2; wolffd@0: wolffd@0: data.rhythm.energy16 = six_en(1:end/2); wolffd@0: data.rhythm.energy16_time = sixt(1:end/2); wolffd@0: wolffd@0: wolffd@0: % save beat interval / tempo wolffd@0: if feature.my_params.norm_interval wolffd@0: wolffd@0: % 1 second max value wolffd@0: data.rhythm.interval16 = eightl / 2; wolffd@0: else wolffd@0: data.rhythm.interval16 = eightl / 2; wolffd@0: end wolffd@0: wolffd@0: end wolffd@0: else wolffd@0: wolffd@0: % % save empty rythm struct wolffd@0: % data.rhythm = struct([]); wolffd@0: end wolffd@0: wolffd@0: % chroma feature data wolffd@0: for i = 1:size(chromas,1) wolffd@0: data.chroma(i).means = chromas(i,:)'; wolffd@0: data.chroma(i).means_weight = cwght(i); wolffd@0: data.chroma(i).vars = chroma_var(i,:)'; wolffd@0: data.chroma(i).shift = shift; wolffd@0: end wolffd@0: wolffd@0: % mfcc feature data wolffd@0: for i = 1:size(mfccs,1) wolffd@0: data.timbre(i).means = mfccs(i,:)'; wolffd@0: data.timbre(i).means_weight = mwght(i); wolffd@0: data.timbre(i).vars = timbre_var(i,:)'; wolffd@0: end wolffd@0: wolffd@0: % prepare field for final features wolffd@0: data.final.vector = []; wolffd@0: data.final.vector_info = struct(); wolffd@0: data.final.dim = 0; wolffd@0: wolffd@0: % save info data wolffd@0: data.info.type = 'MTTAudioFeatureBasicSm'; wolffd@0: data.info.owner = clip; wolffd@0: data.info.owner_id = clip.id; wolffd@0: data.info.creatorrev = feature.my_revision; wolffd@0: wolffd@0: % save parameters wolffd@0: data.info.params = feature.my_params; wolffd@0: end wolffd@0: wolffd@0: function define_global_transform(features) wolffd@0: % calculate and set normalization factors from the group of wolffd@0: % input features. These features will be set for the full database wolffd@0: wolffd@0: if numel(features) == 1 wolffd@0: error ('Insert feature array for this method'); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % here, we only need to define the post-normalisation wolffd@0: % --- wolffd@0: wolffd@0: % --- wolffd@0: % get chroma variance data NORMALISATION Factors wolffd@0: % TODO: transport chroma variance to finalise step wolffd@0: % --- wolffd@0: if features(1).my_params.chroma_var >= 1 wolffd@0: allfeat = abs(cat(2, features(1).data.chroma(:).vars)); wolffd@0: for i = 2:numel(features) wolffd@0: wolffd@0: allfeat = cat(2 , allfeat, abs(abs(cat(2, features(i).data.chroma(:).vars)))); wolffd@0: end wolffd@0: [~, common.post_normf.chroma_var] = mapminmax(allfeat,0,1); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % get timbre variance data NORMALISATION Factors wolffd@0: % TODO: transport chroma variance to finalise step wolffd@0: % --- wolffd@0: if features(1).my_params.timbre_var >= 1 wolffd@0: allfeat = abs(cat(2, features(1).data.timbre(:).vars)); wolffd@0: for i = 2:numel(features) wolffd@0: wolffd@0: allfeat = cat(2 , allfeat, abs(abs(cat(2, features(i).data.timbre(:).vars)))); wolffd@0: end wolffd@0: [~, common.post_normf.timbre_var] = mapminmax(allfeat,0,1); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % derive normalisation for timbre features: wolffd@0: % MFCC's are actually special filter outputs wolffd@0: % (see developer.echonest.com/docs/v4/_static/AnalyzeDocumentation_2.2.pdf wolffd@0: % they are unbounded, so just the relative information will be wolffd@0: % used here. wolffd@0: % We normalise each bin independently wolffd@0: % --- wolffd@0: if features(1).my_params.ntimbres > 0 wolffd@0: wolffd@0: allfeat = abs(cat(2, features(1).data.timbre(:).means)); wolffd@0: for i = 2:numel(features) wolffd@0: wolffd@0: allfeat = cat(2 , allfeat, abs(cat(2, features(i).data.timbre(:).means))); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % get normalisation factors wolffd@0: % NOTE: the values will later be clipped to [0,1] wolffd@0: % anyways wolffd@0: % --- wolffd@0: if (features(1).my_params.clip_timbres ~= 0 ) || ... wolffd@0: (features(1).my_params.clip_timbres ~= 1 ) wolffd@0: wolffd@0: common.post_normf.timbre = 1 ./ prctile(allfeat, features(1).my_params.clip_timbres * 100, 2); wolffd@0: wolffd@0: else wolffd@0: % just use the maximum wolffd@0: common.post_normf.timbre = 1/max(allfeat, 2); wolffd@0: end wolffd@0: wolffd@0: % set common feature values wolffd@0: features(1).my_db.set_common(common); wolffd@0: wolffd@0: else wolffd@0: wolffd@0: features(1).my_db.set_common([1]); wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: wolffd@0: function finalise(feature) wolffd@0: % applies a final transformation and wolffd@0: % collects the information of this feature within a single vector wolffd@0: % see info for types in specific dimensions wolffd@0: wolffd@0: for i = 1:numel(feature) wolffd@0: wolffd@0: % check for neccesary parameters wolffd@0: if isempty(feature(i).my_db.commondb) wolffd@0: wolffd@0: error('Define the global transformation first') wolffd@0: return; wolffd@0: end wolffd@0: wolffd@0: if feature(1).my_params.ntimbres > 0 wolffd@0: % --- wolffd@0: % normalise features wolffd@0: % --- wolffd@0: % norm timbre features if neccesary wolffd@0: timbren = []; wolffd@0: if feature(i).my_params.norm_timbres wolffd@0: for j = 1:numel(feature(i).data.timbre) wolffd@0: wolffd@0: timbren = cat(1, timbren, ... wolffd@0: MTTAudioFeatureBasicSm.norm_timbre... wolffd@0: (feature(i).data.timbre(j).means, feature(i).my_db.commondb.post_normf.timbre)); wolffd@0: end wolffd@0: else wolffd@0: wolffd@0: timbren = cat(1, timbren, feature(i).data.timbre(:).means); wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % construct resulting feature vector out of features wolffd@0: % --- wolffd@0: vec = []; wolffd@0: info = {}; wolffd@0: if feature(i).my_params.nchromas > 0 wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'chroma'; wolffd@0: vec = cat(1, vec, feature(i).data.chroma(:).means); wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'chroma weights'; wolffd@0: vec = cat(1, vec, [feature(i).data.chroma(:).means_weight]'); wolffd@0: wolffd@0: % --- wolffd@0: % NORMALISE Chroma variance wolffd@0: % --- wolffd@0: if feature(i).my_params.chroma_var >= 1 wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'chroma variance'; wolffd@0: wolffd@0: % normalise this pack of variance vectors wolffd@0: tmp_var = mapminmax('apply', [feature(i).data.chroma(:).vars],... wolffd@0: feature(i).common.post_normf.chroma_var); wolffd@0: wolffd@0: % concatenate normalised data to vector wolffd@0: for vari = 1:size(tmp_var,2) wolffd@0: wolffd@0: vec = cat(1, vec, tmp_var(:, vari)); wolffd@0: end wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: wolffd@0: if feature(i).my_params.ntimbres > 0 wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'timbre'; wolffd@0: vec = cat(1, vec, timbren); wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'timbre weights'; wolffd@0: vec = cat(1, vec, [feature(i).data.timbre(:).means_weight]'); wolffd@0: wolffd@0: % --- wolffd@0: % NORMALISE timbre variance wolffd@0: % --- wolffd@0: if feature(i).my_params.timbre_var >= 1 wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'timbre variance'; wolffd@0: wolffd@0: % normalise this pack of variance vectors wolffd@0: tmp_var = mapminmax('apply', [feature(i).data.timbre(:).vars],... wolffd@0: feature(i).common.post_normf.timbre_var); wolffd@0: wolffd@0: % concatenate normalised data to vector wolffd@0: for vari = 1:size(tmp_var,2) wolffd@0: wolffd@0: vec = cat(1, vec, tmp_var(:, vari)); wolffd@0: end wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: if feature(i).my_params.nrhythms > 0 wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'rhythm 8'; wolffd@0: vec = cat(1, vec, feature(i).data.rhythm.acorr8); wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'int 8'; wolffd@0: vec = cat(1, vec, feature(i).data.rhythm.interval8); wolffd@0: wolffd@0: if feature(i).my_params.nrhythms >= 2 wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'rhythm 16'; wolffd@0: vec = cat(1, vec, feature(i).data.rhythm.acorr16); wolffd@0: wolffd@0: info{numel(vec)+ 1} = 'int 16'; wolffd@0: vec = cat(1, vec, feature(i).data.rhythm.interval16); wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: feature(i).data.final.vector = vec; wolffd@0: feature(i).data.final.dim = numel(feature(i).data.final.vector); wolffd@0: wolffd@0: % fill up info struct and append to feature wolffd@0: wolffd@0: info(end+1: feature(i).data.final.dim) = ... wolffd@0: cell(feature(i).data.final.dim - numel(info),1); wolffd@0: wolffd@0: feature(i).data.final.vector_info.labels = info; wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % TODO: Maybe delete more basic features again at this point? wolffd@0: % --- wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % destructor: do we really want to remove this wolffd@0: % from the database? No, but wolffd@0: % TODO: create marker for unused objects in db, and a cleanup wolffd@0: % function wolffd@0: % --- wolffd@0: function delete(feature) wolffd@0: wolffd@0: end wolffd@0: wolffd@0: wolffd@0: function visualise(feature) wolffd@0: % --- wolffd@0: % plots the different data types collected in this feature wolffd@0: % --- wolffd@0: for i = 1:numel(feature) wolffd@0: clip = feature(i).data.info.owner; wolffd@0: wolffd@0: % display raw features wolffd@0: if isa(clip, 'CASIMIRClip') wolffd@0: baseclip = clip.child_clip(); wolffd@0: else wolffd@0: baseclip = clip; wolffd@0: end wolffd@0: if isa(baseclip, 'MTTClip') wolffd@0: rawf = baseclip.audio_features_raw(); wolffd@0: elseif isa(baseclip, 'MSDClip') wolffd@0: rawf = baseclip.features('MSDAudioFeatureRAW'); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % @todo: implement MSD feature visualisation wolffd@0: % --- wolffd@0: [a1, a2, a3] = rawf.visualise(); wolffd@0: wolffd@0: % --- wolffd@0: % Display chroma features wolffd@0: % --- wolffd@0: if isfield(feature(i).data, 'chroma') wolffd@0: wolffd@0: chroma_labels = {'c', 'c#', 'd','d#', 'e', 'f','f#', 'g','g#', 'a', 'a#', 'h'}; wolffd@0: mode_labels = {'minor', 'major'}; wolffd@0: wolffd@0: % change labels to reflect detected mode wolffd@0: chroma_labels{rawf.data.key + 1} = ... wolffd@0: sprintf('(%s) %s',mode_labels{rawf.data.mode + 1}, chroma_labels{rawf.data.key + 1}); wolffd@0: wolffd@0: % transpose labels and data wolffd@0: chroma_labels = circshift(chroma_labels, [0, feature(i).data.chroma(1).shift]); wolffd@0: chromar = circshift([rawf.data.segments_pitches], [feature(i).data.chroma(1).shift, 0]); wolffd@0: wolffd@0: % image transposed chromas again wolffd@0: segments = [rawf.data.segments_start]; wolffd@0: segments(end) = rawf.data.duration; wolffd@0: wolffd@0: hold(a1); wolffd@0: uimagesc(segments, 0:11, chromar, 'Parent', a1); wolffd@0: set(a1,'YTick',[0:11], 'YTickLabel', chroma_labels); wolffd@0: wolffd@0: % enlarge plot and plot new data after the old ones wolffd@0: ax = axis(a1); wolffd@0: ax(2) = ax(2) + 2*feature(i).my_params.nchromas + 0.5; wolffd@0: axis(a1, 'xy'); wolffd@0: axis(a1, ax); wolffd@0: wolffd@0: imagesc(rawf.data.duration + (1:feature(i).my_params.nchromas), (-1:11), ... wolffd@0: [ feature(i).data.chroma(:).means_weight; feature(i).data.chroma(:).means],... wolffd@0: 'Parent', a1); wolffd@0: % variance calculated? wolffd@0: if isfield(feature(i).data.chroma, 'vars') wolffd@0: wolffd@0: imagesc(rawf.data.duration + feature(i).my_params.nchromas + (1:feature(i).my_params.nchromas), (-1:11), ... wolffd@0: [feature(i).data.chroma(:).vars],... wolffd@0: 'Parent', a1); wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % Display timbre features wolffd@0: % --- wolffd@0: if isfield(feature(i).data, 'timbre') wolffd@0: wolffd@0: % enlarge plot and plot new data after the old ones wolffd@0: hold(a2); wolffd@0: ax = axis(a2); wolffd@0: ax(2) = ax(2) + 2*feature(i).my_params.ntimbres + 0.5; wolffd@0: wolffd@0: axis(a2, ax); wolffd@0: imagesc(rawf.data.duration + (1:feature(i).my_params.ntimbres), (-1:11), ... wolffd@0: [ feature(i).data.timbre(:).means_weight; feature(i).data.timbre(:).means],... wolffd@0: 'Parent', a2); wolffd@0: if isfield(feature(i).data.timbre, 'vars') wolffd@0: wolffd@0: imagesc(rawf.data.duration + feature(i).my_params.ntimbres + (1:feature(i).my_params.ntimbres), (-1:11), ... wolffd@0: [feature(i).data.timbre(:).vars],... wolffd@0: 'Parent', a1); wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % Display rhythm features wolffd@0: % --- wolffd@0: if isfield(feature(i).data, 'rhythm') wolffd@0: % data.rhythm.interval wolffd@0: % get timecode wolffd@0: eightt = feature(i).data.rhythm.energy8_time; wolffd@0: sixt = feature(i).data.rhythm.energy16_time; wolffd@0: wolffd@0: hold(a3); wolffd@0: % plot sixteens acorr and energy wolffd@0: plot(sixt, feature(i).data.rhythm.energy16, 'bx') wolffd@0: wolffd@0: plot(sixt, feature(i).data.rhythm.acorr16, 'b') wolffd@0: wolffd@0: % plot eights acorr and energy wolffd@0: plot(eightt, feature(i).data.rhythm.energy8, 'rx') wolffd@0: wolffd@0: plot(eightt, feature(i).data.rhythm.acorr8, 'r') wolffd@0: wolffd@0: % broaden view by fixed 4 seconds wolffd@0: ax = axis(a3); wolffd@0: axis(a3, [max(0, eightt(1)-( eightt(end) - eightt(1) + 4 )) ... wolffd@0: min(rawf.data.duration, eightt(end) +4) ... wolffd@0: ax(3:4)]); wolffd@0: end wolffd@0: end wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: wolffd@0: methods (Hidden = true) wolffd@0: wolffd@0: function [env, time] = energy_envelope(feature, clip) wolffd@0: % extracts the envelope of energy for the given clip wolffd@0: wolffd@0: % --- wolffd@0: % TODO: externalise envelope etc in external audio features wolffd@0: % --- wolffd@0: wolffd@0: [null, src] = evalc('miraudio(clip.mp3file_full())'); wolffd@0: [null, env] = evalc('mirenvelope(src, ''Sampling'', feature.my_params.energy_sr)'); wolffd@0: wolffd@0: time = get(env,'Time'); wolffd@0: time = time{1}{1}; wolffd@0: env = mirgetdata(env); wolffd@0: end wolffd@0: wolffd@0: function [acorr, base_sig, base_t] = beat_histogram(feature, startt, interval, signal, signal_t) wolffd@0: % acorr = beat_histogram(feature, startt, interval, signal, time) wolffd@0: % wolffd@0: % compute correlation for beats of specified length in energy curve wolffd@0: wolffd@0: % get corresponding energy values wolffd@0: dt = signal_t(2) - signal_t(1); wolffd@0: base_t = startt:interval:(startt + (feature.my_params.nints*2-1) * interval); wolffd@0: base_sig = signal( min( numel(signal), max(1,round((base_t - signal_t(1))/dt)))); wolffd@0: wolffd@0: % normalise energy wolffd@0: acbase_sig = base_sig./max(base_sig); wolffd@0: wolffd@0: % calculate their cyclic autocorrelation wolffd@0: acorr = circshift(xcorr(acbase_sig,acbase_sig(1:end/2)),... wolffd@0: [numel(acbase_sig) 0]); wolffd@0: wolffd@0: % cut acorr to relevant points, normalise and square wolffd@0: acorr = (acorr(1:feature.my_params.nints)./feature.my_params.nints).^2; wolffd@0: wolffd@0: % --- wolffd@0: % NOTE: we normalise the autocorrelation locally, to compare the wolffd@0: % (rhythmic) shape wolffd@0: % --- wolffd@0: if feature.my_params.norm_acorr; wolffd@0: wolffd@0: acorr = acorr - min(acorr); wolffd@0: acorr = acorr/max(acorr); wolffd@0: end wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: methods(Static) wolffd@0: wolffd@0: function timbre = norm_timbre(in, normfs) wolffd@0: % returns normed timbre data wolffd@0: wolffd@0: % --- wolffd@0: % individually scale the data using wolffd@0: % the dimensions factors wolffd@0: % --- wolffd@0: timbre = zeros(size(in)); wolffd@0: for i = 1:size(in,2) wolffd@0: wolffd@0: timbre(:,i) = normfs .* in(:,i); wolffd@0: end wolffd@0: wolffd@0: % shift to positive values wolffd@0: timbre = (1 + timbre) /2; wolffd@0: wolffd@0: % clip features to [0,1] wolffd@0: timbre = min(1, max(timbre, 0)); wolffd@0: end wolffd@0: wolffd@0: % --- wolffd@0: % returns parameter md5 hash for comparison wolffd@0: % --- wolffd@0: end wolffd@0: wolffd@0: end