Daniel@0: function varargout = miremotion(orig,varargin) Daniel@0: % Predicts emotion along three dimensions and five basic concepts. Daniel@0: % Optional parameters: Daniel@0: % miremotion(...,'Dimensions',0) excludes all three dimensions. Daniel@0: % miremotion(...,'Dimensions',3) includes all three dimensions (default). Daniel@0: % miremotion(...,'Activity') includes the 'Activity' dimension. Daniel@0: % miremotion(...,'Valence') includes the 'Valence' dimension. Daniel@0: % miremotion(...,'Tension') includes the 'Tension' dimension. Daniel@0: % miremotion(...,'Dimensions',2) includes 'Activity' and 'Valence'. Daniel@0: % miremotion(...,'Arousal') includes 'Activity' and 'Tension'. Daniel@0: % miremotion(...,'Concepts',0) excludes all five concepts. Daniel@0: % miremotion(...,'Concepts') includes all five concepts (default). Daniel@0: % miremotion(...,'Happy') includes the 'Happy' concept. Daniel@0: % miremotion(...,'Sad') includes the 'Sad' concept. Daniel@0: % miremotion(...,'Tender') includes the 'Tender' concept. Daniel@0: % miremotion(...,'Anger') includes the 'Anger' concept. Daniel@0: % miremotion(...,'Fear') includes the 'Fear' concept. Daniel@0: % miremotion(...,'Frame',...) predict emotion frame by frame. Daniel@0: % Daniel@0: % Selection of features and coefficients are taken from a study: Daniel@0: % Eerola, T., Lartillot, O., and Toiviainen, P. Daniel@0: % (2009). Prediction of multidimensional emotional ratings in Daniel@0: % music from audio using multivariate regression models. Daniel@0: % In Proceedings of 10th International Conference on Music Information Retrieval Daniel@0: % (ISMIR 2009), pages 621-626. Daniel@0: % Daniel@0: % The implemented models are based on multiple linear regression with 5 best Daniel@0: % predictors (MLR option in the paper). The box-cox transformations have now been Daniel@0: % removed until the normalization values have been established with a large sample of music. Daniel@0: % Daniel@0: % TODO: Revision of coefficients to (a) force the output range between 0 - 1 and Daniel@0: % (b) to be based on alternative models and materials (training sets). Daniel@0: % Daniel@0: % Updated 03.05.2010 TE Daniel@0: % Daniel@0: frame.key = 'Frame'; Daniel@0: frame.type = 'Integer'; Daniel@0: frame.number = 2; Daniel@0: frame.default = [0 0]; Daniel@0: frame.keydefault = [1 1]; Daniel@0: option.frame = frame; Daniel@0: Daniel@0: dim.key = 'Dimensions'; Daniel@0: dim.type = 'Integer'; Daniel@0: dim.default = NaN; Daniel@0: dim.keydefault = 3; Daniel@0: option.dim = dim; Daniel@0: Daniel@0: activity.key = 'Activity'; Daniel@0: activity.type = 'Boolean'; Daniel@0: activity.default = NaN; Daniel@0: option.activity = activity; Daniel@0: Daniel@0: valence.key = 'Valence'; Daniel@0: valence.type = 'Boolean'; Daniel@0: valence.default = NaN; Daniel@0: option.valence = valence; Daniel@0: Daniel@0: tension.key = 'Tension'; Daniel@0: tension.type = 'Boolean'; Daniel@0: tension.default = NaN; Daniel@0: option.tension = tension; Daniel@0: Daniel@0: arousal.key = 'Arousal'; Daniel@0: arousal.type = 'Boolean'; Daniel@0: arousal.default = NaN; Daniel@0: option.arousal = arousal; Daniel@0: Daniel@0: concepts.key = 'Concepts'; Daniel@0: concepts.type = 'Boolean'; Daniel@0: concepts.default = NaN; Daniel@0: option.concepts = concepts; Daniel@0: Daniel@0: happy.key = 'Happy'; Daniel@0: happy.type = 'Boolean'; Daniel@0: happy.default = NaN; Daniel@0: option.happy = happy; Daniel@0: Daniel@0: sad.key = 'Sad'; Daniel@0: sad.type = 'Boolean'; Daniel@0: sad.default = NaN; Daniel@0: option.sad = sad; Daniel@0: Daniel@0: tender.key = 'Tender'; Daniel@0: tender.type = 'Boolean'; Daniel@0: tender.default = NaN; Daniel@0: option.tender = tender; Daniel@0: Daniel@0: anger.key = 'Anger'; Daniel@0: anger.type = 'Boolean'; Daniel@0: anger.default = NaN; Daniel@0: option.anger = anger; Daniel@0: Daniel@0: fear.key = 'Fear'; Daniel@0: fear.type = 'Boolean'; Daniel@0: fear.default = NaN; Daniel@0: option.fear = fear; Daniel@0: Daniel@0: specif.option = option; Daniel@0: specif.defaultframelength = 1; Daniel@0: %specif.defaultframehop = .5; Daniel@0: Daniel@0: specif.combinechunk = {'Average',@nothing}; Daniel@0: specif.extensive = 1; Daniel@0: Daniel@0: varargout = mirfunction(@miremotion,orig,varargin,nargout,specif,@init,@main); Daniel@0: Daniel@0: Daniel@0: %% Daniel@0: function [x type] = init(x,option) Daniel@0: Daniel@0: option = process(option); Daniel@0: Daniel@0: if option.frame.length.val Daniel@0: hop = option.frame.hop.val; Daniel@0: if strcmpi(option.frame.hop.unit,'Hz') Daniel@0: hop = 1/hop; Daniel@0: option.frame.hop.unit = 's'; Daniel@0: end Daniel@0: if strcmpi(option.frame.hop.unit,'s') Daniel@0: hop = hop*get(x,'Sampling'); Daniel@0: end Daniel@0: if strcmpi(option.frame.hop.unit,'%') Daniel@0: hop = hop/100; Daniel@0: option.frame.hop.unit = '/1'; Daniel@0: end Daniel@0: if strcmpi(option.frame.hop.unit,'/1') Daniel@0: hop = hop*option.frame.length.val; Daniel@0: end Daniel@0: frames = 0:hop:1000000; Daniel@0: x = mirsegment(x,[frames;frames+option.frame.length.val]); Daniel@0: elseif isa(x,'mirdesign') Daniel@0: x = set(x,'NoChunk',1); Daniel@0: end Daniel@0: rm = mirrms(x,'Frame',.046,.5); Daniel@0: Daniel@0: le = 0; %mirlowenergy(rm,'ASR'); Daniel@0: Daniel@0: o = mironsets(x,'Filterbank',15,'Contrast',0.1); Daniel@0: at = mirattacktime(o); Daniel@0: as = 0; %mirattackslope(o); Daniel@0: ed = 0; %mireventdensity(o,'Option1'); Daniel@0: Daniel@0: fl = mirfluctuation(x,'Summary'); Daniel@0: fp = mirpeaks(fl,'Total',1); Daniel@0: fc = 0; %mircentroid(fl); Daniel@0: Daniel@0: tp = 0; %mirtempo(x,'Frame',2,.5,'Autocor','Spectrum'); Daniel@0: pc = mirpulseclarity(x,'Frame',2,.5); %%%%%%%%%%% Why 'Frame'?? Daniel@0: Daniel@0: s = mirspectrum(x,'Frame',.046,.5); Daniel@0: sc = mircentroid(s); Daniel@0: ss = mirspread(s); Daniel@0: sr = mirroughness(s); Daniel@0: Daniel@0: %ps = mirpitch(x,'Frame',.046,.5,'Tolonen'); Daniel@0: Daniel@0: c = mirchromagram(x,'Frame','Wrap',0,'Pitch',0); %%%%%%%%%%%%%%%%%%%% Previous frame size was too small. Daniel@0: cp = mirpeaks(c,'Total',1); Daniel@0: ps = 0;%cp; Daniel@0: ks = mirkeystrength(c); Daniel@0: [k kc] = mirkey(ks); Daniel@0: mo = mirmode(ks); Daniel@0: hc = mirhcdf(c); Daniel@0: Daniel@0: se = mirentropy(mirspectrum(x,'Collapsed','Min',40,'Smooth',70,'Frame',1.5,.5)); %%%%%%%%% Why 'Frame'?? Daniel@0: Daniel@0: ns = mirnovelty(mirspectrum(x,'Frame',.1,.5,'Max',5000),'Normal',0); Daniel@0: nt = mirnovelty(mirchromagram(x,'Frame',.2,.25),'Normal',0); %%%%%%%%%%%%%%%%%%%% Previous frame size was too small. Daniel@0: nr = mirnovelty(mirchromagram(x,'Frame',.2,.25,'Wrap',0),'Normal',0); %%%%%%%%%%%%%%%%%%%% Previous frame size was too small. Daniel@0: Daniel@0: Daniel@0: Daniel@0: x = {rm,le, at,as,ed, fp,fc, tp,pc, sc,ss,sr, ps, cp,kc,mo,hc, se, ns,nt,nr}; Daniel@0: Daniel@0: type = {'miremotion','mirscalar','mirscalar',... Daniel@0: 'mirscalar','mirscalar','mirscalar',... Daniel@0: 'mirspectrum','mirscalar',... Daniel@0: 'mirscalar','mirscalar',... Daniel@0: 'mirscalar','mirscalar','mirscalar',... Daniel@0: 'mirscalar',... Daniel@0: 'mirchromagram','mirscalar','mirscalar','mirscalar',... Daniel@0: 'mirscalar',... Daniel@0: 'mirscalar','mirscalar','mirscalar'}; Daniel@0: Daniel@0: Daniel@0: %% Daniel@0: function e = main(x,option,postoption) Daniel@0: Daniel@0: warning('WARNING IN MIRENOTION: The current model of miremotion is not correctly calibrated with this version of MIRtoolbox (but with version 1.3 only).'); Daniel@0: Daniel@0: option = process(option); Daniel@0: rm = get(x{1},'Data'); Daniel@0: %le = get(x{2},'Data'); Daniel@0: at = get(x{3},'Data'); Daniel@0: %as = get(x{4},'Data'); Daniel@0: %ed = get(x{5},'Data'); Daniel@0: %fpp = get(x{6},'PeakPosUnit'); Daniel@0: fpv = get(x{6},'PeakVal'); Daniel@0: %fc = get(x{7},'Data'); Daniel@0: %tp = get(x{8},'Data'); Daniel@0: pc = get(x{9},'Data'); Daniel@0: sc = get(x{10},'Data'); Daniel@0: ss = get(x{11},'Data'); Daniel@0: rg = get(x{12},'Data'); Daniel@0: %ps = get(x{13},'PeakPosUnit'); Daniel@0: cp = get(x{14},'PeakPosUnit'); Daniel@0: kc = get(x{15},'Data'); Daniel@0: mo = get(x{16},'Data'); Daniel@0: hc = get(x{17},'Data'); Daniel@0: se = get(x{18},'Data'); Daniel@0: ns = get(x{19},'Data'); Daniel@0: nt = get(x{20},'Data'); Daniel@0: nr = get(x{21},'Data'); Daniel@0: Daniel@0: Daniel@0: e.dim = {}; Daniel@0: e.dimdata = mircompute(@initialise,rm); Daniel@0: if option.activity == 1 Daniel@0: [e.dimdata e.activity_fact] = mircompute(@activity,e.dimdata,rm,fpv,sc,ss,se); Daniel@0: e.dim = [e.dim,'Activity']; Daniel@0: else Daniel@0: e.activity_fact = NaN; Daniel@0: end Daniel@0: if option.valence == 1 Daniel@0: [e.dimdata e.valence_fact] = mircompute(@valence,e.dimdata,rm,fpv,kc,mo,ns); Daniel@0: e.dim = [e.dim,'Valence']; Daniel@0: else Daniel@0: e.valence_fact = NaN; Daniel@0: end Daniel@0: if option.tension == 1 Daniel@0: [e.dimdata e.tension_fact] = mircompute(@tension,e.dimdata,rm,fpv,kc,hc,nr); Daniel@0: e.dim = [e.dim,'Tension']; Daniel@0: else Daniel@0: e.tension_fact = NaN; Daniel@0: end Daniel@0: Daniel@0: e.class = {}; Daniel@0: e.classdata = mircompute(@initialise,rm); Daniel@0: if option.happy == 1 Daniel@0: [e.classdata e.happy_fact] = mircompute(@happy,e.classdata,fpv,ss,cp,kc,mo); Daniel@0: e.class = [e.class,'Happy']; Daniel@0: else Daniel@0: e.happy_fact = NaN; Daniel@0: end Daniel@0: if option.sad == 1 Daniel@0: [e.classdata e.sad_fact] = mircompute(@sad,e.classdata,ss,cp,mo,hc,nt); Daniel@0: e.class = [e.class,'Sad']; Daniel@0: else Daniel@0: e.sad_fact = NaN; Daniel@0: end Daniel@0: if option.tender == 1 Daniel@0: [e.classdata e.tender_fact] = mircompute(@tender,e.classdata,sc,rg,kc,hc,ns); Daniel@0: e.class = [e.class,'Tender']; Daniel@0: else Daniel@0: e.tender_fact = NaN; Daniel@0: end Daniel@0: if option.anger == 1 Daniel@0: [e.classdata e.anger_fact] = mircompute(@anger,e.classdata,rg,kc,se,nr); Daniel@0: e.class = [e.class,'Anger']; Daniel@0: else Daniel@0: e.anger_fact = NaN; Daniel@0: end Daniel@0: if option.fear == 1 Daniel@0: [e.classdata e.fear_fact] = mircompute(@fear,e.classdata,rm,at,fpv,kc,mo); Daniel@0: e.class = [e.class,'Fear']; Daniel@0: else Daniel@0: e.fear_fact = NaN; Daniel@0: end Daniel@0: Daniel@0: e = class(e,'miremotion',mirdata(x{1})); Daniel@0: e = purgedata(e); Daniel@0: fp = mircompute(@noframe,get(x{1},'FramePos')); Daniel@0: e = set(e,'Title','Emotion','Abs','emotions','Ord','magnitude','FramePos',fp); Daniel@0: Daniel@0: %% Daniel@0: function option = process(option) Daniel@0: if option.arousal==1 Daniel@0: option.activity = 1; Daniel@0: option.tension = 1; Daniel@0: if isnan(option.dim) Daniel@0: option.dim = 0; Daniel@0: end Daniel@0: end Daniel@0: if option.activity==1 || option.valence==1 || option.tension==1 Daniel@0: if isnan(option.activity) Daniel@0: option.activity = 0; Daniel@0: end Daniel@0: if isnan(option.valence) Daniel@0: option.valence = 0; Daniel@0: end Daniel@0: if isnan(option.tension) Daniel@0: option.tension = 0; Daniel@0: end Daniel@0: if isnan(option.concepts) Daniel@0: option.concepts = 0; Daniel@0: end Daniel@0: end Daniel@0: if not(isnan(option.dim)) && option.dim Daniel@0: if isnan(option.concepts) Daniel@0: option.concepts = 0; Daniel@0: end Daniel@0: end Daniel@0: if not(isnan(option.concepts)) && option.concepts Daniel@0: if isnan(option.dim) Daniel@0: option.dim = 0; Daniel@0: end Daniel@0: end Daniel@0: if not(isnan(option.dim)) Daniel@0: switch option.dim Daniel@0: case 0 Daniel@0: if isnan(option.activity) Daniel@0: option.activity = 0; Daniel@0: end Daniel@0: if isnan(option.valence) Daniel@0: option.valence = 0; Daniel@0: end Daniel@0: if isnan(option.tension) Daniel@0: option.tension = 0; Daniel@0: end Daniel@0: case 2 Daniel@0: option.activity = 1; Daniel@0: option.valence = 1; Daniel@0: if isnan(option.tension) Daniel@0: option.tension = 0; Daniel@0: end Daniel@0: case 3 Daniel@0: option.activity = 1; Daniel@0: option.valence = 1; Daniel@0: option.tension = 1; Daniel@0: end Daniel@0: end Daniel@0: if isnan(option.activity) Daniel@0: option.activity = 1; Daniel@0: end Daniel@0: if isnan(option.valence) Daniel@0: option.valence = 1; Daniel@0: end Daniel@0: if isnan(option.tension) Daniel@0: option.tension = 1; Daniel@0: end Daniel@0: if isnan(option.concepts) Daniel@0: option.concepts = 1; Daniel@0: end Daniel@0: if option.concepts Daniel@0: option.happy = 1; Daniel@0: option.sad = 1; Daniel@0: option.tender = 1; Daniel@0: option.anger = 1; Daniel@0: option.fear = 1; Daniel@0: end Daniel@0: if option.happy==1 || option.sad==1 || option.tender==1 ... Daniel@0: || option.anger==1 || option.fear==1 Daniel@0: if isnan(option.happy) Daniel@0: option.happy = 0; Daniel@0: end Daniel@0: if isnan(option.sad) Daniel@0: option.sad = 0; Daniel@0: end Daniel@0: if isnan(option.tender) Daniel@0: option.tender = 0; Daniel@0: end Daniel@0: if isnan(option.anger) Daniel@0: option.anger = 0; Daniel@0: end Daniel@0: if isnan(option.fear) Daniel@0: option.fear = 0; Daniel@0: end Daniel@0: end Daniel@0: Daniel@0: Daniel@0: %% Daniel@0: function e = initialise(rm) Daniel@0: e = []; Daniel@0: Daniel@0: Daniel@0: function e = activity(e,rm,fpv,sc,ss,se) % without the box-cox transformation, revised coefficients Daniel@0: af = zeros(5,1); Daniel@0: af(1) = 0.6664* ((mean(rm) - 0.0559)/0.0337); % Daniel@0: af(2) = 0.6099 * ((mean(fpv{1}) - 13270.1836)/10790.655); Daniel@0: af(3) = 0.4486*((mean(cell2mat(sc)) - 1677.7)./570.34); Daniel@0: af(4) = -0.4639*((mean(cell2mat(ss)) - 250.5574)./205.3147); Daniel@0: af(5) = 0.7056*((mean(se) - 0.954)./0.0258); Daniel@0: af(isnan(af)) = []; Daniel@0: e(end+1,:) = sum(af)+5.4861; Daniel@0: e = {e af}; Daniel@0: Daniel@0: function e = valence(e,rm,fpv,kc,mo,ns) % without the box-cox transformation, revised coefficients Daniel@0: vf = zeros(5,1); Daniel@0: vf(1) = -0.3161 * ((std(rm) - 0.024254)./0.015667); Daniel@0: vf(2) = 0.6099 * ((mean(fpv{1}) - 13270.1836)/10790.655); Daniel@0: vf(3) = 0.8802 * ((mean(kc) - 0.5123)./0.091953); Daniel@0: vf(4) = 0.4565 * ((mean(mo) - -0.0019958)./0.048664); Daniel@0: ns(isnan(ns)) = []; Daniel@0: vf(5) = 0.4015 * ((mean(ns) - 131.9503)./47.6463); Daniel@0: vf(isnan(vf)) = []; Daniel@0: e(end+1,:) = sum(vf)+5.2749; Daniel@0: e = {e vf}; Daniel@0: Daniel@0: function e = tension(e,rm,fpv,kc,hc,nr) Daniel@0: tf = zeros(5,1); Daniel@0: tf(1) = 0.5382 * ((std(rm) - 0.024254)./0.015667); Daniel@0: tf(2) = -0.5406 * ((mean(fpv{1}) - 13270.1836)/10790.655); Daniel@0: tf(3) = -0.6808 * ((mean(kc) - 0.5124)./0.092); Daniel@0: tf(4) = 0.8629 * ((mean(hc) - 0.2962)./0.0459); Daniel@0: tf(5) = -0.5958 * ((mean(nr) - 71.8426)./46.9246); Daniel@0: tf(isnan(tf)) = []; Daniel@0: e(end+1,:) = sum(tf)+5.4679; Daniel@0: e = {e tf}; Daniel@0: Daniel@0: Daniel@0: % BASIC EMOTION PREDICTORS Daniel@0: Daniel@0: function e = happy(e,fpv,ss,cp,kc,mo) Daniel@0: ha_f = zeros(5,1); Daniel@0: ha_f(1) = 0.7438*((mean(cell2mat(fpv)) - 13270.1836)./10790.655); Daniel@0: ha_f(2) = -0.3965*((mean(cell2mat(ss)) - 250.5574)./205.3147); Daniel@0: ha_f(3) = 0.4047*((std(cell2mat(cp)) - 8.5321)./2.5899); Daniel@0: ha_f(4) = 0.7780*((mean(kc) - 0.5124)./0.092); Daniel@0: ha_f(5) = 0.6220*((mean(mo) - -0.002)./0.0487); Daniel@0: ha_f(isnan(ha_f)) = []; Daniel@0: e(end+1,:) = sum(ha_f)+2.6166; Daniel@0: e = {e ha_f}; Daniel@0: Daniel@0: function e = sad(e,ss,cp,mo,hc,nt) Daniel@0: sa_f = zeros(5,1); Daniel@0: sa_f(1) = 0.4324*((mean(cell2mat(ss)) - 250.5574)./205.3147); Daniel@0: sa_f(2) = -0.3137*((std(cell2mat(cp)) - 8.5321)./2.5899); Daniel@0: sa_f(3) = -0.5201*((mean(mo) - -0.0020)./0.0487); Daniel@0: sa_f(4) = -0.6017*((mean(hc) - 0.2962)./0.0459); Daniel@0: sa_f(5) = 0.4493*((mean(nt) - 42.2022)./36.7782); Daniel@0: sa_f(isnan(sa_f)) = []; Daniel@0: e(end+1,:) = sum(sa_f)+2.9756; Daniel@0: e = {e sa_f}; Daniel@0: Daniel@0: function e = tender(e,sc,rg,kc,hc,ns) Daniel@0: te_f = zeros(5,1); Daniel@0: te_f(1) = -0.2709*((mean(cell2mat(sc)) - 1677.7106)./570.3432); Daniel@0: te_f(2) = -0.4904*((std(rg) - 85.9387)./106.0767); Daniel@0: te_f(3) = 0.5192*((mean(kc) - 0.5124)./0.0920); Daniel@0: te_f(4) = -0.3995*((mean(hc) - 0.2962)./0.0459); Daniel@0: te_f(5) = 0.3391*((mean(ns) - 131.9503)./47.6463); Daniel@0: te_f(isnan(te_f)) = []; Daniel@0: e(end+1,:) = sum(te_f)+2.9756; Daniel@0: e = {e te_f}; Daniel@0: Daniel@0: function e = anger(e,rg,kc,se,nr) % Daniel@0: an_f = zeros(5,1); Daniel@0: %an_f(1) = -0.2353*((mean(pc) - 0.1462)./.1113); Daniel@0: an_f(2) = 0.5517*((mean(rg) - 85.9387)./106.0767); Daniel@0: an_f(3) = -.5802*((mean(kc) - 0.5124)./0.092); Daniel@0: an_f(4) = .2821*((mean(se) - 0.954)./0.0258); Daniel@0: an_f(5) = -.2971*((mean(nr) - 71.8426)./46.9246); Daniel@0: an_f(isnan(an_f)) = []; Daniel@0: e(end+1,:) = sum(an_f)+1.9767; Daniel@0: e = {e an_f}; Daniel@0: Daniel@0: function e = fear(e,rm,at,fpv,kc,mo) Daniel@0: fe_f = zeros(5,1); Daniel@0: fe_f(1) = 0.4069*((std(rm) - 0.0243)./0.0157); Daniel@0: fe_f(2) = -0.6388*((mean(at) - 0.0707)./0.015689218536423); Daniel@0: fe_f(3) = -0.2538*((mean(cell2mat(fpv)) - 13270.1836)./10790.655); Daniel@0: fe_f(4) = -0.9860*((mean(kc) - 0.5124)./0.0920); Daniel@0: fe_f(5) = -0.3144*((mean(mo) - -0.0019958)./0.048663550639094); Daniel@0: fe_f(isnan(fe_f)) = []; Daniel@0: e(end+1,:) = sum(fe_f)+2.7847; Daniel@0: e = {e fe_f}; Daniel@0: Daniel@0: function fp = noframe(fp) Daniel@0: fp = [fp(1);fp(end)];