wolffd@0: function varargout = miremotion(orig,varargin) wolffd@0: % Predicts emotion along three dimensions and five basic concepts. wolffd@0: % Optional parameters: wolffd@0: % miremotion(...,'Dimensions',0) excludes all three dimensions. wolffd@0: % miremotion(...,'Dimensions',3) includes all three dimensions (default). wolffd@0: % miremotion(...,'Activity') includes the 'Activity' dimension. wolffd@0: % miremotion(...,'Valence') includes the 'Valence' dimension. wolffd@0: % miremotion(...,'Tension') includes the 'Tension' dimension. wolffd@0: % miremotion(...,'Dimensions',2) includes 'Activity' and 'Valence'. wolffd@0: % miremotion(...,'Arousal') includes 'Activity' and 'Tension'. wolffd@0: % miremotion(...,'Concepts',0) excludes all five concepts. wolffd@0: % miremotion(...,'Concepts') includes all five concepts (default). wolffd@0: % miremotion(...,'Happy') includes the 'Happy' concept. wolffd@0: % miremotion(...,'Sad') includes the 'Sad' concept. wolffd@0: % miremotion(...,'Tender') includes the 'Tender' concept. wolffd@0: % miremotion(...,'Anger') includes the 'Anger' concept. wolffd@0: % miremotion(...,'Fear') includes the 'Fear' concept. wolffd@0: % miremotion(...,'Frame',...) predict emotion frame by frame. wolffd@0: % wolffd@0: % Selection of features and coefficients are taken from a study: wolffd@0: % Eerola, T., Lartillot, O., and Toiviainen, P. wolffd@0: % (2009). Prediction of multidimensional emotional ratings in wolffd@0: % music from audio using multivariate regression models. wolffd@0: % In Proceedings of 10th International Conference on Music Information Retrieval wolffd@0: % (ISMIR 2009), pages 621-626. wolffd@0: % wolffd@0: % The implemented models are based on multiple linear regression with 5 best wolffd@0: % predictors (MLR option in the paper). The box-cox transformations have now been wolffd@0: % removed until the normalization values have been established with a large sample of music. wolffd@0: % wolffd@0: % TODO: Revision of coefficients to (a) force the output range between 0 - 1 and wolffd@0: % (b) to be based on alternative models and materials (training sets). wolffd@0: % wolffd@0: % Updated 03.05.2010 TE wolffd@0: % wolffd@0: frame.key = 'Frame'; wolffd@0: frame.type = 'Integer'; wolffd@0: frame.number = 2; wolffd@0: frame.default = [0 0]; wolffd@0: frame.keydefault = [1 1]; wolffd@0: option.frame = frame; wolffd@0: wolffd@0: dim.key = 'Dimensions'; wolffd@0: dim.type = 'Integer'; wolffd@0: dim.default = NaN; wolffd@0: dim.keydefault = 3; wolffd@0: option.dim = dim; wolffd@0: wolffd@0: activity.key = 'Activity'; wolffd@0: activity.type = 'Boolean'; wolffd@0: activity.default = NaN; wolffd@0: option.activity = activity; wolffd@0: wolffd@0: valence.key = 'Valence'; wolffd@0: valence.type = 'Boolean'; wolffd@0: valence.default = NaN; wolffd@0: option.valence = valence; wolffd@0: wolffd@0: tension.key = 'Tension'; wolffd@0: tension.type = 'Boolean'; wolffd@0: tension.default = NaN; wolffd@0: option.tension = tension; wolffd@0: wolffd@0: arousal.key = 'Arousal'; wolffd@0: arousal.type = 'Boolean'; wolffd@0: arousal.default = NaN; wolffd@0: option.arousal = arousal; wolffd@0: wolffd@0: concepts.key = 'Concepts'; wolffd@0: concepts.type = 'Boolean'; wolffd@0: concepts.default = NaN; wolffd@0: option.concepts = concepts; wolffd@0: wolffd@0: happy.key = 'Happy'; wolffd@0: happy.type = 'Boolean'; wolffd@0: happy.default = NaN; wolffd@0: option.happy = happy; wolffd@0: wolffd@0: sad.key = 'Sad'; wolffd@0: sad.type = 'Boolean'; wolffd@0: sad.default = NaN; wolffd@0: option.sad = sad; wolffd@0: wolffd@0: tender.key = 'Tender'; wolffd@0: tender.type = 'Boolean'; wolffd@0: tender.default = NaN; wolffd@0: option.tender = tender; wolffd@0: wolffd@0: anger.key = 'Anger'; wolffd@0: anger.type = 'Boolean'; wolffd@0: anger.default = NaN; wolffd@0: option.anger = anger; wolffd@0: wolffd@0: fear.key = 'Fear'; wolffd@0: fear.type = 'Boolean'; wolffd@0: fear.default = NaN; wolffd@0: option.fear = fear; wolffd@0: wolffd@0: specif.option = option; wolffd@0: specif.defaultframelength = 1; wolffd@0: %specif.defaultframehop = .5; wolffd@0: wolffd@0: specif.combinechunk = {'Average',@nothing}; wolffd@0: specif.extensive = 1; wolffd@0: wolffd@0: varargout = mirfunction(@miremotion,orig,varargin,nargout,specif,@init,@main); wolffd@0: wolffd@0: wolffd@0: %% wolffd@0: function [x type] = init(x,option) wolffd@0: wolffd@0: option = process(option); wolffd@0: wolffd@0: if option.frame.length.val wolffd@0: hop = option.frame.hop.val; wolffd@0: if strcmpi(option.frame.hop.unit,'Hz') wolffd@0: hop = 1/hop; wolffd@0: option.frame.hop.unit = 's'; wolffd@0: end wolffd@0: if strcmpi(option.frame.hop.unit,'s') wolffd@0: hop = hop*get(x,'Sampling'); wolffd@0: end wolffd@0: if strcmpi(option.frame.hop.unit,'%') wolffd@0: hop = hop/100; wolffd@0: option.frame.hop.unit = '/1'; wolffd@0: end wolffd@0: if strcmpi(option.frame.hop.unit,'/1') wolffd@0: hop = hop*option.frame.length.val; wolffd@0: end wolffd@0: frames = 0:hop:1000000; wolffd@0: x = mirsegment(x,[frames;frames+option.frame.length.val]); wolffd@0: elseif isa(x,'mirdesign') wolffd@0: x = set(x,'NoChunk',1); wolffd@0: end wolffd@0: rm = mirrms(x,'Frame',.046,.5); wolffd@0: wolffd@0: le = 0; %mirlowenergy(rm,'ASR'); wolffd@0: wolffd@0: o = mironsets(x,'Filterbank',15,'Contrast',0.1); wolffd@0: at = mirattacktime(o); wolffd@0: as = 0; %mirattackslope(o); wolffd@0: ed = 0; %mireventdensity(o,'Option1'); wolffd@0: wolffd@0: fl = mirfluctuation(x,'Summary'); wolffd@0: fp = mirpeaks(fl,'Total',1); wolffd@0: fc = 0; %mircentroid(fl); wolffd@0: wolffd@0: tp = 0; %mirtempo(x,'Frame',2,.5,'Autocor','Spectrum'); wolffd@0: pc = mirpulseclarity(x,'Frame',2,.5); %%%%%%%%%%% Why 'Frame'?? wolffd@0: wolffd@0: s = mirspectrum(x,'Frame',.046,.5); wolffd@0: sc = mircentroid(s); wolffd@0: ss = mirspread(s); wolffd@0: sr = mirroughness(s); wolffd@0: wolffd@0: %ps = mirpitch(x,'Frame',.046,.5,'Tolonen'); wolffd@0: wolffd@0: c = mirchromagram(x,'Frame','Wrap',0,'Pitch',0); %%%%%%%%%%%%%%%%%%%% Previous frame size was too small. wolffd@0: cp = mirpeaks(c,'Total',1); wolffd@0: ps = 0;%cp; wolffd@0: ks = mirkeystrength(c); wolffd@0: [k kc] = mirkey(ks); wolffd@0: mo = mirmode(ks); wolffd@0: hc = mirhcdf(c); wolffd@0: wolffd@0: se = mirentropy(mirspectrum(x,'Collapsed','Min',40,'Smooth',70,'Frame',1.5,.5)); %%%%%%%%% Why 'Frame'?? wolffd@0: wolffd@0: ns = mirnovelty(mirspectrum(x,'Frame',.1,.5,'Max',5000),'Normal',0); wolffd@0: nt = mirnovelty(mirchromagram(x,'Frame',.2,.25),'Normal',0); %%%%%%%%%%%%%%%%%%%% Previous frame size was too small. wolffd@0: nr = mirnovelty(mirchromagram(x,'Frame',.2,.25,'Wrap',0),'Normal',0); %%%%%%%%%%%%%%%%%%%% Previous frame size was too small. wolffd@0: wolffd@0: wolffd@0: wolffd@0: x = {rm,le, at,as,ed, fp,fc, tp,pc, sc,ss,sr, ps, cp,kc,mo,hc, se, ns,nt,nr}; wolffd@0: wolffd@0: type = {'miremotion','mirscalar','mirscalar',... wolffd@0: 'mirscalar','mirscalar','mirscalar',... wolffd@0: 'mirspectrum','mirscalar',... wolffd@0: 'mirscalar','mirscalar',... wolffd@0: 'mirscalar','mirscalar','mirscalar',... wolffd@0: 'mirscalar',... wolffd@0: 'mirchromagram','mirscalar','mirscalar','mirscalar',... wolffd@0: 'mirscalar',... wolffd@0: 'mirscalar','mirscalar','mirscalar'}; wolffd@0: wolffd@0: wolffd@0: %% wolffd@0: function e = main(x,option,postoption) wolffd@0: wolffd@0: warning('WARNING IN MIRENOTION: The current model of miremotion is not correctly calibrated with this version of MIRtoolbox (but with version 1.3 only).'); wolffd@0: wolffd@0: option = process(option); wolffd@0: rm = get(x{1},'Data'); wolffd@0: %le = get(x{2},'Data'); wolffd@0: at = get(x{3},'Data'); wolffd@0: %as = get(x{4},'Data'); wolffd@0: %ed = get(x{5},'Data'); wolffd@0: %fpp = get(x{6},'PeakPosUnit'); wolffd@0: fpv = get(x{6},'PeakVal'); wolffd@0: %fc = get(x{7},'Data'); wolffd@0: %tp = get(x{8},'Data'); wolffd@0: pc = get(x{9},'Data'); wolffd@0: sc = get(x{10},'Data'); wolffd@0: ss = get(x{11},'Data'); wolffd@0: rg = get(x{12},'Data'); wolffd@0: %ps = get(x{13},'PeakPosUnit'); wolffd@0: cp = get(x{14},'PeakPosUnit'); wolffd@0: kc = get(x{15},'Data'); wolffd@0: mo = get(x{16},'Data'); wolffd@0: hc = get(x{17},'Data'); wolffd@0: se = get(x{18},'Data'); wolffd@0: ns = get(x{19},'Data'); wolffd@0: nt = get(x{20},'Data'); wolffd@0: nr = get(x{21},'Data'); wolffd@0: wolffd@0: wolffd@0: e.dim = {}; wolffd@0: e.dimdata = mircompute(@initialise,rm); wolffd@0: if option.activity == 1 wolffd@0: [e.dimdata e.activity_fact] = mircompute(@activity,e.dimdata,rm,fpv,sc,ss,se); wolffd@0: e.dim = [e.dim,'Activity']; wolffd@0: else wolffd@0: e.activity_fact = NaN; wolffd@0: end wolffd@0: if option.valence == 1 wolffd@0: [e.dimdata e.valence_fact] = mircompute(@valence,e.dimdata,rm,fpv,kc,mo,ns); wolffd@0: e.dim = [e.dim,'Valence']; wolffd@0: else wolffd@0: e.valence_fact = NaN; wolffd@0: end wolffd@0: if option.tension == 1 wolffd@0: [e.dimdata e.tension_fact] = mircompute(@tension,e.dimdata,rm,fpv,kc,hc,nr); wolffd@0: e.dim = [e.dim,'Tension']; wolffd@0: else wolffd@0: e.tension_fact = NaN; wolffd@0: end wolffd@0: wolffd@0: e.class = {}; wolffd@0: e.classdata = mircompute(@initialise,rm); wolffd@0: if option.happy == 1 wolffd@0: [e.classdata e.happy_fact] = mircompute(@happy,e.classdata,fpv,ss,cp,kc,mo); wolffd@0: e.class = [e.class,'Happy']; wolffd@0: else wolffd@0: e.happy_fact = NaN; wolffd@0: end wolffd@0: if option.sad == 1 wolffd@0: [e.classdata e.sad_fact] = mircompute(@sad,e.classdata,ss,cp,mo,hc,nt); wolffd@0: e.class = [e.class,'Sad']; wolffd@0: else wolffd@0: e.sad_fact = NaN; wolffd@0: end wolffd@0: if option.tender == 1 wolffd@0: [e.classdata e.tender_fact] = mircompute(@tender,e.classdata,sc,rg,kc,hc,ns); wolffd@0: e.class = [e.class,'Tender']; wolffd@0: else wolffd@0: e.tender_fact = NaN; wolffd@0: end wolffd@0: if option.anger == 1 wolffd@0: [e.classdata e.anger_fact] = mircompute(@anger,e.classdata,rg,kc,se,nr); wolffd@0: e.class = [e.class,'Anger']; wolffd@0: else wolffd@0: e.anger_fact = NaN; wolffd@0: end wolffd@0: if option.fear == 1 wolffd@0: [e.classdata e.fear_fact] = mircompute(@fear,e.classdata,rm,at,fpv,kc,mo); wolffd@0: e.class = [e.class,'Fear']; wolffd@0: else wolffd@0: e.fear_fact = NaN; wolffd@0: end wolffd@0: wolffd@0: e = class(e,'miremotion',mirdata(x{1})); wolffd@0: e = purgedata(e); wolffd@0: fp = mircompute(@noframe,get(x{1},'FramePos')); wolffd@0: e = set(e,'Title','Emotion','Abs','emotions','Ord','magnitude','FramePos',fp); wolffd@0: wolffd@0: %% wolffd@0: function option = process(option) wolffd@0: if option.arousal==1 wolffd@0: option.activity = 1; wolffd@0: option.tension = 1; wolffd@0: if isnan(option.dim) wolffd@0: option.dim = 0; wolffd@0: end wolffd@0: end wolffd@0: if option.activity==1 || option.valence==1 || option.tension==1 wolffd@0: if isnan(option.activity) wolffd@0: option.activity = 0; wolffd@0: end wolffd@0: if isnan(option.valence) wolffd@0: option.valence = 0; wolffd@0: end wolffd@0: if isnan(option.tension) wolffd@0: option.tension = 0; wolffd@0: end wolffd@0: if isnan(option.concepts) wolffd@0: option.concepts = 0; wolffd@0: end wolffd@0: end wolffd@0: if not(isnan(option.dim)) && option.dim wolffd@0: if isnan(option.concepts) wolffd@0: option.concepts = 0; wolffd@0: end wolffd@0: end wolffd@0: if not(isnan(option.concepts)) && option.concepts wolffd@0: if isnan(option.dim) wolffd@0: option.dim = 0; wolffd@0: end wolffd@0: end wolffd@0: if not(isnan(option.dim)) wolffd@0: switch option.dim wolffd@0: case 0 wolffd@0: if isnan(option.activity) wolffd@0: option.activity = 0; wolffd@0: end wolffd@0: if isnan(option.valence) wolffd@0: option.valence = 0; wolffd@0: end wolffd@0: if isnan(option.tension) wolffd@0: option.tension = 0; wolffd@0: end wolffd@0: case 2 wolffd@0: option.activity = 1; wolffd@0: option.valence = 1; wolffd@0: if isnan(option.tension) wolffd@0: option.tension = 0; wolffd@0: end wolffd@0: case 3 wolffd@0: option.activity = 1; wolffd@0: option.valence = 1; wolffd@0: option.tension = 1; wolffd@0: end wolffd@0: end wolffd@0: if isnan(option.activity) wolffd@0: option.activity = 1; wolffd@0: end wolffd@0: if isnan(option.valence) wolffd@0: option.valence = 1; wolffd@0: end wolffd@0: if isnan(option.tension) wolffd@0: option.tension = 1; wolffd@0: end wolffd@0: if isnan(option.concepts) wolffd@0: option.concepts = 1; wolffd@0: end wolffd@0: if option.concepts wolffd@0: option.happy = 1; wolffd@0: option.sad = 1; wolffd@0: option.tender = 1; wolffd@0: option.anger = 1; wolffd@0: option.fear = 1; wolffd@0: end wolffd@0: if option.happy==1 || option.sad==1 || option.tender==1 ... wolffd@0: || option.anger==1 || option.fear==1 wolffd@0: if isnan(option.happy) wolffd@0: option.happy = 0; wolffd@0: end wolffd@0: if isnan(option.sad) wolffd@0: option.sad = 0; wolffd@0: end wolffd@0: if isnan(option.tender) wolffd@0: option.tender = 0; wolffd@0: end wolffd@0: if isnan(option.anger) wolffd@0: option.anger = 0; wolffd@0: end wolffd@0: if isnan(option.fear) wolffd@0: option.fear = 0; wolffd@0: end wolffd@0: end wolffd@0: wolffd@0: wolffd@0: %% wolffd@0: function e = initialise(rm) wolffd@0: e = []; wolffd@0: wolffd@0: wolffd@0: function e = activity(e,rm,fpv,sc,ss,se) % without the box-cox transformation, revised coefficients wolffd@0: af = zeros(5,1); wolffd@0: af(1) = 0.6664* ((mean(rm) - 0.0559)/0.0337); % wolffd@0: af(2) = 0.6099 * ((mean(fpv{1}) - 13270.1836)/10790.655); wolffd@0: af(3) = 0.4486*((mean(cell2mat(sc)) - 1677.7)./570.34); wolffd@0: af(4) = -0.4639*((mean(cell2mat(ss)) - 250.5574)./205.3147); wolffd@0: af(5) = 0.7056*((mean(se) - 0.954)./0.0258); wolffd@0: af(isnan(af)) = []; wolffd@0: e(end+1,:) = sum(af)+5.4861; wolffd@0: e = {e af}; wolffd@0: wolffd@0: function e = valence(e,rm,fpv,kc,mo,ns) % without the box-cox transformation, revised coefficients wolffd@0: vf = zeros(5,1); wolffd@0: vf(1) = -0.3161 * ((std(rm) - 0.024254)./0.015667); wolffd@0: vf(2) = 0.6099 * ((mean(fpv{1}) - 13270.1836)/10790.655); wolffd@0: vf(3) = 0.8802 * ((mean(kc) - 0.5123)./0.091953); wolffd@0: vf(4) = 0.4565 * ((mean(mo) - -0.0019958)./0.048664); wolffd@0: ns(isnan(ns)) = []; wolffd@0: vf(5) = 0.4015 * ((mean(ns) - 131.9503)./47.6463); wolffd@0: vf(isnan(vf)) = []; wolffd@0: e(end+1,:) = sum(vf)+5.2749; wolffd@0: e = {e vf}; wolffd@0: wolffd@0: function e = tension(e,rm,fpv,kc,hc,nr) wolffd@0: tf = zeros(5,1); wolffd@0: tf(1) = 0.5382 * ((std(rm) - 0.024254)./0.015667); wolffd@0: tf(2) = -0.5406 * ((mean(fpv{1}) - 13270.1836)/10790.655); wolffd@0: tf(3) = -0.6808 * ((mean(kc) - 0.5124)./0.092); wolffd@0: tf(4) = 0.8629 * ((mean(hc) - 0.2962)./0.0459); wolffd@0: tf(5) = -0.5958 * ((mean(nr) - 71.8426)./46.9246); wolffd@0: tf(isnan(tf)) = []; wolffd@0: e(end+1,:) = sum(tf)+5.4679; wolffd@0: e = {e tf}; wolffd@0: wolffd@0: wolffd@0: % BASIC EMOTION PREDICTORS wolffd@0: wolffd@0: function e = happy(e,fpv,ss,cp,kc,mo) wolffd@0: ha_f = zeros(5,1); wolffd@0: ha_f(1) = 0.7438*((mean(cell2mat(fpv)) - 13270.1836)./10790.655); wolffd@0: ha_f(2) = -0.3965*((mean(cell2mat(ss)) - 250.5574)./205.3147); wolffd@0: ha_f(3) = 0.4047*((std(cell2mat(cp)) - 8.5321)./2.5899); wolffd@0: ha_f(4) = 0.7780*((mean(kc) - 0.5124)./0.092); wolffd@0: ha_f(5) = 0.6220*((mean(mo) - -0.002)./0.0487); wolffd@0: ha_f(isnan(ha_f)) = []; wolffd@0: e(end+1,:) = sum(ha_f)+2.6166; wolffd@0: e = {e ha_f}; wolffd@0: wolffd@0: function e = sad(e,ss,cp,mo,hc,nt) wolffd@0: sa_f = zeros(5,1); wolffd@0: sa_f(1) = 0.4324*((mean(cell2mat(ss)) - 250.5574)./205.3147); wolffd@0: sa_f(2) = -0.3137*((std(cell2mat(cp)) - 8.5321)./2.5899); wolffd@0: sa_f(3) = -0.5201*((mean(mo) - -0.0020)./0.0487); wolffd@0: sa_f(4) = -0.6017*((mean(hc) - 0.2962)./0.0459); wolffd@0: sa_f(5) = 0.4493*((mean(nt) - 42.2022)./36.7782); wolffd@0: sa_f(isnan(sa_f)) = []; wolffd@0: e(end+1,:) = sum(sa_f)+2.9756; wolffd@0: e = {e sa_f}; wolffd@0: wolffd@0: function e = tender(e,sc,rg,kc,hc,ns) wolffd@0: te_f = zeros(5,1); wolffd@0: te_f(1) = -0.2709*((mean(cell2mat(sc)) - 1677.7106)./570.3432); wolffd@0: te_f(2) = -0.4904*((std(rg) - 85.9387)./106.0767); wolffd@0: te_f(3) = 0.5192*((mean(kc) - 0.5124)./0.0920); wolffd@0: te_f(4) = -0.3995*((mean(hc) - 0.2962)./0.0459); wolffd@0: te_f(5) = 0.3391*((mean(ns) - 131.9503)./47.6463); wolffd@0: te_f(isnan(te_f)) = []; wolffd@0: e(end+1,:) = sum(te_f)+2.9756; wolffd@0: e = {e te_f}; wolffd@0: wolffd@0: function e = anger(e,rg,kc,se,nr) % wolffd@0: an_f = zeros(5,1); wolffd@0: %an_f(1) = -0.2353*((mean(pc) - 0.1462)./.1113); wolffd@0: an_f(2) = 0.5517*((mean(rg) - 85.9387)./106.0767); wolffd@0: an_f(3) = -.5802*((mean(kc) - 0.5124)./0.092); wolffd@0: an_f(4) = .2821*((mean(se) - 0.954)./0.0258); wolffd@0: an_f(5) = -.2971*((mean(nr) - 71.8426)./46.9246); wolffd@0: an_f(isnan(an_f)) = []; wolffd@0: e(end+1,:) = sum(an_f)+1.9767; wolffd@0: e = {e an_f}; wolffd@0: wolffd@0: function e = fear(e,rm,at,fpv,kc,mo) wolffd@0: fe_f = zeros(5,1); wolffd@0: fe_f(1) = 0.4069*((std(rm) - 0.0243)./0.0157); wolffd@0: fe_f(2) = -0.6388*((mean(at) - 0.0707)./0.015689218536423); wolffd@0: fe_f(3) = -0.2538*((mean(cell2mat(fpv)) - 13270.1836)./10790.655); wolffd@0: fe_f(4) = -0.9860*((mean(kc) - 0.5124)./0.0920); wolffd@0: fe_f(5) = -0.3144*((mean(mo) - -0.0019958)./0.048663550639094); wolffd@0: fe_f(isnan(fe_f)) = []; wolffd@0: e(end+1,:) = sum(fe_f)+2.7847; wolffd@0: e = {e fe_f}; wolffd@0: wolffd@0: function fp = noframe(fp) wolffd@0: fp = [fp(1);fp(end)];