Maria@4: # -*- coding: utf-8 -*-
Maria@4: """
Maria@4: Created on Thu Mar 16 01:50:57 2017
Maria@4: 
Maria@4: @author: mariapanteli
Maria@4: """
Maria@4: 
Maria@4: import numpy as np
Maria@4: import pandas as pd
Maria@4: import os
Maria@4: from sklearn.decomposition import NMF
Maria@4: import OPMellin as opm
Maria@4: import MFCC as mfc
Maria@4: import PitchBihist as pbi
Maria@4: 
Maria@4: 
Maria@4: class FeatureLoader:
Maria@4:     def __init__(self, win2sec=8):
Maria@4:         self.win2sec = float(win2sec)
Maria@4:         self.sr = 44100.
Maria@4:         self.win1 = int(round(0.04*self.sr))
Maria@4:         self.hop1 = int(round(self.win1/8.))
Maria@4:         self.framessr = self.sr/float(self.hop1)
Maria@4:         self.win2 = int(round(self.win2sec*self.framessr))
Maria@4:         self.hop2 = int(round(0.5*self.framessr))
Maria@4:         self.framessr2 = self.framessr/float(self.hop2)
Maria@4:     
Maria@4:     
m@26:     def get_op_mfcc_for_file(self, melspec_file=None, scale=True, stop_sec=30.0):
Maria@4:         op = []
Maria@4:         mfc = []
Maria@4:         if not os.path.exists(melspec_file):
Maria@4:             return op, mfc
Maria@4:         print 'extracting onset patterns and mfccs...'
Maria@4:         songframes = pd.read_csv(melspec_file, engine="c", header=None)
Maria@4:         songframes.iloc[np.where(np.isnan(songframes))] = 0
Maria@35:         n_stop = np.int(np.ceil(stop_sec * self.framessr))
m@26:         songframes = songframes.iloc[0:min(len(songframes), n_stop), :]
Maria@4:         melspec = songframes.get_values().T
Maria@4:         op = self.get_op_from_melspec(melspec, K=2)
Maria@4:         mfc = self.get_mfcc_from_melspec(melspec)
Maria@4:         if scale:
Maria@4:             # scale all frames by mean and std of recording
Maria@4:             op = (op - np.nanmean(op)) / np.nanstd(op) 
Maria@4:             mfc = (mfc - np.nanmean(mfc)) / np.nanstd(mfc)
Maria@4:         return op, mfc
Maria@4:     
Maria@4:     
m@26:     def get_chroma_for_file(self, chroma_file=None, scale=True, stop_sec=30.0):
Maria@4:         ch = []
Maria@4:         if not os.path.exists(chroma_file):
Maria@4:             return ch
Maria@4:         print 'extracting chroma...'
Maria@4:         songframes = pd.read_csv(chroma_file, engine="c", header=None)
Maria@4:         songframes.iloc[np.where(np.isnan(songframes))] = 0
m@26:         n_stop = np.int(np.ceil(stop_sec * self.framessr))
Maria@35:         songframes = songframes.iloc[0:min(len(songframes), n_stop), :]
Maria@4:         chroma = songframes.get_values().T
Maria@4:         ch = self.get_ave_chroma(chroma)
Maria@4:         if scale:
Maria@4:             # scale all frames by mean and std of recording
Maria@4:             ch = (ch - np.nanmean(ch)) / np.nanstd(ch)                        
Maria@4:         return ch
Maria@4:     
Maria@4:         
Maria@4:     def get_music_idx_from_bounds(self, bounds, sr=None):
Maria@4:         music_idx = []
Maria@4:         if len(bounds) == 0:  
Maria@4:             # bounds is empty list
Maria@4:             return music_idx
Maria@4:         nbounds = bounds.shape[0]
Maria@4:         if len(np.where(bounds[:,2]=='m')[0])==0:
Maria@4:             # no music segments
Maria@4:             return music_idx
Maria@4:         elif len(np.where(bounds[:,2]=='s')[0])==nbounds:
Maria@4:             # all segments are speech
Maria@4:             return music_idx
Maria@4:         else:
Maria@34:             win2_frames = np.int(np.round(self.win2sec * self.framessr2))
Maria@34:             #half_win_hop = int(round(0.5 * self.win2 / float(self.hop2)))
Maria@4:             music_bounds = np.where(bounds[:, 2] == 'm')[0]
Maria@4:             bounds_in_frames = np.round(np.array(bounds[:, 0], dtype=float) * sr)
Maria@34:             duration_in_frames = np.ceil(np.array(bounds[:, 1], dtype=float) * sr)
Maria@4:             for music_bound in music_bounds:
Maria@34:                 #lower_bound = np.max([0, bounds_in_frames[music_bound] - half_win_hop])
Maria@34:                 #upper_bound = bounds_in_frames[music_bound] + duration_in_frames[music_bound] - half_win_hop
Maria@34:                 lower_bound = bounds_in_frames[music_bound]
Maria@34:                 upper_bound = bounds_in_frames[music_bound] + duration_in_frames[music_bound] - win2_frames
Maria@4:                 music_idx.append(np.arange(lower_bound, upper_bound, dtype=int))
Maria@4:             if len(music_idx)>0:
Maria@4:                 music_idx = np.sort(np.concatenate(music_idx))  # it should be sorted, but just in case segments overlap -- remove duplicates if segments overlap
Maria@4:         return music_idx
m@6: 
Maria@4:     
Maria@4:     def get_music_idx_for_file(self, segmenter_file=None):
Maria@4:         music_idx = []
Maria@4:         if os.path.exists(segmenter_file) and os.path.getsize(segmenter_file)>0:
Maria@4:             print 'loading speech/music segments...'
Maria@4:             bounds = pd.read_csv(segmenter_file, header=None, delimiter='\t').get_values()
Maria@4:             if bounds.shape[1] == 1:  # depends on the computer platform
Maria@4:                 bounds = pd.read_csv(segmenter_file, header=None, delimiter=',').get_values()    
Maria@4:             music_idx = self.get_music_idx_from_bounds(bounds, sr=self.framessr2)
Maria@4:         return music_idx
Maria@4:     
Maria@4:     
m@42:     def get_features(self, df, stop_sec=30.0, class_label='Country', precomp_melody=True):
Maria@4:         oplist = []
Maria@4:         mflist = []
Maria@4:         chlist = []
Maria@4:         pblist = []        
Maria@4:         clabels = []
Maria@4:         aulabels = []
Maria@4:         n_files = len(df)
Maria@4:         for i in range(n_files):
Maria@4:             if not (os.path.exists(df['Melspec'].iloc[i]) and os.path.exists(df['Chroma'].iloc[i]) and os.path.exists(df['Melodia'].iloc[i])):
Maria@4:                 continue
Maria@4:             print 'file ' + str(i) + ' of ' + str(n_files)
Maria@4:             music_idx = self.get_music_idx_for_file(df['Speech'].iloc[i])
m@43:             #min_dur_sec=8.0
m@43:             #min_n_frames = np.int(np.floor(min_dur_sec * self.framessr2))
m@43:             if len(music_idx)==0:  # or len(music_idx)<min_n_frames:
m@43:                 # no music segments or music segment too short -> skip this file
Maria@4:                 continue
m@46:             try:
m@43:                 # allow feature extraction of longer segments (2*stop_sec)
m@43:                 # because some of it might be speech segments that are filtered out
m@43:                 stop_sec_feat = 2 * stop_sec  
m@43:                 op, mfcc = self.get_op_mfcc_for_file(df['Melspec'].iloc[i], stop_sec=stop_sec_feat)
m@43:                 ch = self.get_chroma_for_file(df['Chroma'].iloc[i], stop_sec=stop_sec_feat)
m@43:                 pb = self.get_pb_for_file(df['Melodia'].iloc[i], precomp_melody=precomp_melody, stop_sec=stop_sec_feat)
Maria@35:                 #if precomp_melody:
Maria@35:                 #    pb = self.load_precomputed_pb_from_melodia(df['Melodia'].iloc[i], stop_sec=stop_sec)
Maria@35:                 #else:
Maria@35:                 #    pb = self.get_pb_from_melodia(df['Melodia'].iloc[i], stop_sec=stop_sec)
m@46:             except:
Maria@4:                 continue
m@26:             n_stop = np.int(np.ceil(stop_sec * self.framessr2))
Maria@35:             print n_stop, len(op), len(mfcc), len(ch), len(pb)
m@43:             max_n_frames = np.min([n_stop, len(op), len(mfcc), len(ch), len(pb)])  # ideally, features should have the same number of frames
m@43:             if max_n_frames==0:
Maria@4:                 # no features extracted -> skip this file
Maria@4:                 continue
m@43:             # music segment duration must be <= 30sec
m@43:             music_idx = music_idx[music_idx<max_n_frames]  
Maria@4:             n_frames = len(music_idx)
Maria@4:             oplist.append(op.iloc[music_idx, :])
Maria@4:             mflist.append(mfcc.iloc[music_idx, :])
Maria@4:             chlist.append(ch.iloc[music_idx, :])
Maria@4:             pblist.append(pb.iloc[music_idx, :])
Maria@4:             clabels.append(pd.DataFrame(np.repeat(df[class_label].iloc[i], n_frames)))
Maria@4:             aulabels.append(pd.DataFrame(np.repeat(df['Audio'].iloc[i], n_frames)))
Maria@4:         print len(oplist), len(mflist), len(chlist), len(pblist), len(clabels), len(aulabels)
Maria@4:         return pd.concat(oplist), pd.concat(mflist), pd.concat(chlist), pd.concat(pblist), pd.concat(clabels), pd.concat(aulabels)
Maria@4: 
Maria@4:             
Maria@4:     def get_op_from_melspec(self, melspec, K=None):
Maria@4:         op = opm.OPMellin(win2sec=self.win2sec)
Maria@4:         opmellin = op.get_opmellin_from_melspec(melspec=melspec, melsr=self.framessr)
Maria@35:         opmel = pd.DataFrame(opmellin.T)
Maria@4:         if K is not None:
Maria@4:             opmel =  self.mean_K_bands(opmellin.T, K)
Maria@35:             opmel = pd.DataFrame(opmel)
Maria@4:         return opmel
Maria@4: 
Maria@4: 
Maria@4:     def get_mfcc_from_melspec(self, melspec, deltamfcc=True, avelocalframes=True, stdlocalframes=True):
Maria@4:         mf = mfc.MFCCs()        
Maria@4:         mfcc = mf.get_mfccs_from_melspec(melspec=melspec, melsr=self.framessr)
Maria@4:         if deltamfcc:
Maria@4:             ff = mfcc
Maria@4:             ffdiff = np.diff(ff, axis=1)
Maria@4:             ffdelta = np.concatenate((ffdiff, ffdiff[:,-1,None]), axis=1)
Maria@4:             frames = np.concatenate([ff,ffdelta], axis=0)                
Maria@4:             mfcc = frames
Maria@4:         if avelocalframes:
Maria@4:             mfcc = self.average_local_frames(mfcc, getstd=stdlocalframes)
Maria@4:         mfcc = pd.DataFrame(mfcc.T)
Maria@4:         return mfcc
Maria@4: 
Maria@4: 
Maria@35:     def get_ave_chroma(self, chroma, alignchroma=True, avelocalframes=True, stdlocalframes=True):
Maria@4:         chroma[np.where(np.isnan(chroma))] = 0
Maria@4:         if alignchroma:
Maria@35:             maxind = np.argmax(np.sum(chroma, axis=1))  # bin with max magnitude across time
Maria@4:             chroma = np.roll(chroma, -maxind, axis=0)
Maria@4:         if avelocalframes:
Maria@4:             chroma = self.average_local_frames(chroma, getstd=stdlocalframes)
Maria@4:         chroma = pd.DataFrame(chroma.T)
Maria@4:         return chroma
Maria@4:   
Maria@4:     
Maria@4:     def average_local_frames(self, frames, getstd=False):
Maria@4:         nbins, norigframes = frames.shape
Maria@4:         if norigframes<self.win2:
Maria@4:             nframes = 1
Maria@4:         else:
Maria@4:             nframes = int(1+np.floor((norigframes-self.win2)/float(self.hop2)))
Maria@4:         if getstd:
Maria@4:             aveframes = np.empty((nbins+nbins, nframes))
Maria@4:             for i in range(nframes):  # loop over all 8-sec frames
Maria@4:                 meanf = np.nanmean(frames[:, (i*self.hop2):min((i*self.hop2+self.win2),norigframes)], axis=1)
Maria@4:                 stdf = np.nanstd(frames[:, (i*self.hop2):min((i*self.hop2+self.win2),norigframes)], axis=1)
Maria@4:                 aveframes[:,i] = np.concatenate((meanf,stdf))
Maria@4:         else:
Maria@4:             aveframes = np.empty((nbins, nframes))
Maria@4:             for i in range(nframes):  # loop over all 8-sec frames
Maria@4:                 aveframes[:,i] = np.nanmean(frames[:, (i*self.hop2):min((i*self.hop2+self.win2),norigframes)], axis=1)
Maria@4:         return aveframes
Maria@4:     
Maria@4:     
Maria@4:     def mean_K_bands(self, songframes, K=40, nmels=40):
Maria@4:         [F, P] = songframes.shape
Maria@4:         Pproc = int((P/nmels)*K)
Maria@4:         procframes = np.zeros([F, Pproc])
Maria@4:         niters = int(nmels/K)
Maria@4:         nbins = P/nmels  # must be 200 bins
Maria@4:         for k in range(K):
Maria@4:             for j in range(k*niters, (k+1)*niters):
Maria@4:                 procframes[:, (k*nbins):((k+1)*nbins)] += songframes[:, (j*nbins):((j+1)*nbins)]
Maria@4:         procframes /= float(niters)
Maria@4:         return procframes
Maria@4:         
Maria@4:         
Maria@4:     def nmfpitchbihist(self, frames):
Maria@4:         nbins, nfr = frames.shape
Maria@4:         npc = 2
Maria@4:         nb = int(np.sqrt(nbins))  # assume structure of pitch bihist is nbins*nbins
Maria@4:         newframes = np.empty(((nb+nb)*npc, nfr))
Maria@4:         for fr in range(nfr):
Maria@4:             pb = np.reshape(frames[:, fr], (nb, nb))
Maria@4:             try:
Maria@4:                 nmfmodel = NMF(n_components=npc).fit(pb)
Maria@4:                 W = nmfmodel.transform(pb)
Maria@4:                 H = nmfmodel.components_.T
Maria@4:                 newframes[:, fr, None] = np.concatenate((W, H)).flatten()[:, None]
Maria@4:             except:
Maria@4:                 newframes[:, fr, None] = np.zeros(((nb+nb)*npc, 1))
Maria@4:         return newframes    
Maria@4: 
Maria@4: 
Maria@35:     def get_pb_for_file(self, melodia_file, precomp_melody=False, nmfpb=True, scale=True, stop_sec=30.0):
Maria@35:         pbihist = []
Maria@35:         if precomp_melody:
Maria@35:             pbihist = self.load_precomp_pb_from_melodia(melodia_file=melodia_file, stop_sec=stop_sec)
Maria@35:         else:
Maria@35:             pbihist = self.extract_pb_from_melodia(melodia_file=melodia_file, stop_sec=stop_sec)
Maria@35:         if len(pbihist) == 0:
Maria@35:             # no file was found
Maria@35:             return pbihist
Maria@4:         if nmfpb is True:
Maria@4:             pbihist = self.nmfpitchbihist(pbihist)
Maria@4:         pbihist = pd.DataFrame(pbihist.T)
Maria@4:         if scale:
Maria@4:             # scale all frames by mean and std of recording
Maria@4:             pbihist = (pbihist - np.nanmean(pbihist)) / np.nanstd(pbihist)
Maria@4:         return pbihist
Maria@4: 
Maria@4: 
Maria@35:     def extract_pb_from_melodia(self, melodia_file=None, stop_sec=30.0):
Maria@35:         pbihist = []
Maria@35:         if not os.path.exists(melodia_file):
Maria@35:             return pbihist
Maria@35:         print 'extracting pitch bihist from melodia...'
Maria@35:         pb = pbi.PitchBihist(win2sec=self.win2sec)
Maria@35:         pbihist = pb.bihist_from_melodia(filename=melodia_file, stop_sec=stop_sec)
Maria@35:         return pbihist
Maria@35: 
Maria@35: 
Maria@35:     def load_precomp_pb_from_melodia(self, melodia_file=None, stop_sec=30.0):
Maria@35:         pbihist = []
Maria@4:         base = os.path.basename(melodia_file)    
Maria@4:         root = '/import/c4dm-05/mariap/Melodia-melody-'+str(int(self.win2sec))+'sec/'
m@24:         root_BL = '/import/c4dm-04/mariap/FeatureCsvs_BL_old/PB-melodia/'
m@24:         root_SM = '/import/c4dm-04/mariap/FeatureCsvs/PB-melodia/'
m@24:         if 'SampleAudio' in base:
m@24:             root = root_SM
m@24:         else:
m@24:             root = root_BL
m@46:             base = base.split('_')[-1].split('.csv')[0]+'_vamp_mtg-melodia_melodia_melody.csv'
m@43:         bihist_file = os.path.join(root, base)
m@43:         if not os.path.exists(bihist_file):
m@43:             return pbihist
Maria@4:         print 'load precomputed pitch bihist', root
m@43:         pbihist = np.loadtxt(bihist_file, delimiter=',').T
Maria@35:         n_stop = np.int(np.ceil(stop_sec * self.framessr2))
Maria@35:         pbihist = pbihist[:, :np.min([pbihist.shape[0], n_stop])]
Maria@4:         return pbihist
Maria@4: