diff scripts/load_features.py @ 35:c4428589b82b branch-tests

more tests in load_features

--- a/scripts/load_features.py	Thu Sep 14 10:16:59 2017 +0100
+++ b/scripts/load_features.py	Thu Sep 14 13:07:19 2017 +0100
@@ -34,7 +34,7 @@
         print 'extracting onset patterns and mfccs...'
         songframes = pd.read_csv(melspec_file, engine="c", header=None)
         songframes.iloc[np.where(np.isnan(songframes))] = 0
-	n_stop = np.int(np.ceil(stop_sec * self.framessr))
+        n_stop = np.int(np.ceil(stop_sec * self.framessr))
         songframes = songframes.iloc[0:min(len(songframes), n_stop), :]
         melspec = songframes.get_values().T
         op = self.get_op_from_melspec(melspec, K=2)
@@ -54,7 +54,7 @@
         songframes = pd.read_csv(chroma_file, engine="c", header=None)
         songframes.iloc[np.where(np.isnan(songframes))] = 0
         n_stop = np.int(np.ceil(stop_sec * self.framessr))
-	songframes = songframes.iloc[0:min(len(songframes), n_stop), :]
+        songframes = songframes.iloc[0:min(len(songframes), n_stop), :]
         chroma = songframes.get_values().T
         ch = self.get_ave_chroma(chroma)
         if scale:
@@ -103,7 +103,7 @@
         return music_idx
     
     
-    def get_features(self, df, stop_sec=30.0, class_label='Country'):
+    def get_features(self, df, stop_sec=30.0, class_label='Country', precomp_melody=False):
         oplist = []
         mflist = []
         chlist = []
@@ -122,14 +122,16 @@
             try:
                 op, mfcc = self.get_op_mfcc_for_file(df['Melspec'].iloc[i], stop_sec=stop_sec)
                 ch = self.get_chroma_for_file(df['Chroma'].iloc[i], stop_sec=stop_sec)
-                #pb = self.get_pb_from_melodia(df['Melodia'].iloc[i], stop_sec=stop_sec)
-                pb = self.load_precomputed_pb_from_melodia(df['Melodia'].iloc[i], stop_sec=stop_sec)
-                #pb = self.get_contour_feat_from_melodia(df['Melodia'].iloc[i])
+                pb = self.get_pb_for_file(df['Melodia'].iloc[i], precomp_melody=precomp_melody, stop_sec=stop_sec)
+                #if precomp_melody:
+                #    pb = self.load_precomputed_pb_from_melodia(df['Melodia'].iloc[i], stop_sec=stop_sec)
+                #else:
+                #    pb = self.get_pb_from_melodia(df['Melodia'].iloc[i], stop_sec=stop_sec)
             except:
                 continue
             n_stop = np.int(np.ceil(stop_sec * self.framessr2))
-	    print n_stop, len(op), len(mfcc), len(ch), len(pb)
-	    min_n_frames = np.min([n_stop, len(op), len(mfcc), len(ch), len(pb)])  # ideally, features should have the same number of frames
+            print n_stop, len(op), len(mfcc), len(ch), len(pb)
+            min_n_frames = np.min([n_stop, len(op), len(mfcc), len(ch), len(pb)])  # ideally, features should have the same number of frames
             if min_n_frames==0:
                 # no features extracted -> skip this file
                 continue
@@ -148,9 +150,10 @@
     def get_op_from_melspec(self, melspec, K=None):
         op = opm.OPMellin(win2sec=self.win2sec)
         opmellin = op.get_opmellin_from_melspec(melspec=melspec, melsr=self.framessr)
+        opmel = pd.DataFrame(opmellin.T)
         if K is not None:
             opmel =  self.mean_K_bands(opmellin.T, K)
-        opmel = pd.DataFrame(opmel)
+            opmel = pd.DataFrame(opmel)
         return opmel
 
 
@@ -169,10 +172,10 @@
         return mfcc
 
 
-    def get_ave_chroma(self, chroma, avelocalframes=True, stdlocalframes=True, alignchroma=True):
+    def get_ave_chroma(self, chroma, alignchroma=True, avelocalframes=True, stdlocalframes=True):
         chroma[np.where(np.isnan(chroma))] = 0
         if alignchroma:
-            maxind = np.argmax(np.sum(chroma, axis=1))
+            maxind = np.argmax(np.sum(chroma, axis=1))  # bin with max magnitude across time
             chroma = np.roll(chroma, -maxind, axis=0)
         if avelocalframes:
             chroma = self.average_local_frames(chroma, getstd=stdlocalframes)
@@ -229,13 +232,15 @@
         return newframes    
 
 
-    def get_pb_from_melodia(self, melodia_file=None, nmfpb=True, scale=True, stop_sec=30.0):
-        pb = []
-        if not os.path.exists(melodia_file):
-            return pb
-        print 'extracting pitch bihist from melodia...'
-        pb = pbi.PitchBihist(win2sec=self.win2sec)
-        pbihist = pb.bihist_from_melodia(filename=melodia_file, stop_sec=stop_sec)
+    def get_pb_for_file(self, melodia_file, precomp_melody=False, nmfpb=True, scale=True, stop_sec=30.0):
+        pbihist = []
+        if precomp_melody:
+            pbihist = self.load_precomp_pb_from_melodia(melodia_file=melodia_file, stop_sec=stop_sec)
+        else:
+            pbihist = self.extract_pb_from_melodia(melodia_file=melodia_file, stop_sec=stop_sec)
+        if len(pbihist) == 0:
+            # no file was found
+            return pbihist
         if nmfpb is True:
             pbihist = self.nmfpitchbihist(pbihist)
         pbihist = pd.DataFrame(pbihist.T)
@@ -245,7 +250,18 @@
         return pbihist
 
 
-    def load_precomputed_pb_from_melodia(self, melodia_file=None, nmfpb=True, scale=True, stop_sec=30.0):
+    def extract_pb_from_melodia(self, melodia_file=None, stop_sec=30.0):
+        pbihist = []
+        if not os.path.exists(melodia_file):
+            return pbihist
+        print 'extracting pitch bihist from melodia...'
+        pb = pbi.PitchBihist(win2sec=self.win2sec)
+        pbihist = pb.bihist_from_melodia(filename=melodia_file, stop_sec=stop_sec)
+        return pbihist
+
+
+    def load_precomp_pb_from_melodia(self, melodia_file=None, stop_sec=30.0):
+        pbihist = []
         base = os.path.basename(melodia_file)    
         root = '/import/c4dm-05/mariap/Melodia-melody-'+str(int(self.win2sec))+'sec/'
         root_BL = '/import/c4dm-04/mariap/FeatureCsvs_BL_old/PB-melodia/'
@@ -254,21 +270,53 @@
             root = root_SM
         else:
             root = root_BL
-	    base = base.split('_')[-1].split('.csv')[0]+'_vamp_mtg-melodia_melodia_melody.csv'
+        base = base.split('_')[-1].split('.csv')[0]+'_vamp_mtg-melodia_melodia_melody.csv'
         print 'load precomputed pitch bihist', root
-        #if self.win2sec == 8:
-        #    pbihist = pd.read_csv(os.path.join(root, base))
-        #else:
-	if 1:
-            pbihist = np.loadtxt(os.path.join(root, base), delimiter=',').T
-            if nmfpb is True:
-                pbihist = self.nmfpitchbihist(pbihist)
-            pbihist = pd.DataFrame(pbihist.T)
-	n_stop = np.int(np.ceil(stop_sec * self.framessr2))
-	pbihist = pbihist.iloc[:np.min([pbihist.shape[0], n_stop]), :]
-        print pbihist.shape
-        if scale:
-            # scale all frames by mean and std of recording
-            pbihist = (pbihist - np.nanmean(pbihist)) / np.nanstd(pbihist)
+        pbihist = np.loadtxt(os.path.join(root, base), delimiter=',').T
+        n_stop = np.int(np.ceil(stop_sec * self.framessr2))
+        pbihist = pbihist[:, :np.min([pbihist.shape[0], n_stop])]
         return pbihist
+
+
+    # def get_pb_from_melodia(self, melodia_file=None, nmfpb=True, scale=True, stop_sec=30.0):
+    #     if not os.path.exists(melodia_file):
+    #         return []
+    #     print 'extracting pitch bihist from melodia...'
+    #     pb = pbi.PitchBihist(win2sec=self.win2sec)
+    #     pbihist = pb.bihist_from_melodia(filename=melodia_file, stop_sec=stop_sec)
+    #     if nmfpb is True:
+    #         pbihist = self.nmfpitchbihist(pbihist)
+    #     pbihist = pd.DataFrame(pbihist.T)
+    #     if scale:
+    #         # scale all frames by mean and std of recording
+    #         pbihist = (pbihist - np.nanmean(pbihist)) / np.nanstd(pbihist)
+    #     return pbihist
+
+
+    # def load_precomputed_pb_from_melodia(self, melodia_file=None, nmfpb=True, scale=True, stop_sec=30.0):
+    #     base = os.path.basename(melodia_file)    
+    #     root = '/import/c4dm-05/mariap/Melodia-melody-'+str(int(self.win2sec))+'sec/'
+    #     root_BL = '/import/c4dm-04/mariap/FeatureCsvs_BL_old/PB-melodia/'
+    #     root_SM = '/import/c4dm-04/mariap/FeatureCsvs/PB-melodia/'
+    #     if 'SampleAudio' in base:
+    #         root = root_SM
+    #     else:
+    #         root = root_BL
+    #     base = base.split('_')[-1].split('.csv')[0]+'_vamp_mtg-melodia_melodia_melody.csv'
+    #     print 'load precomputed pitch bihist', root
+    #     #if self.win2sec == 8:
+    #     #    pbihist = pd.read_csv(os.path.join(root, base))
+    #     #else:
+    #     if 1:
+    #         pbihist = np.loadtxt(os.path.join(root, base), delimiter=',').T
+    #         if nmfpb is True:
+    #             pbihist = self.nmfpitchbihist(pbihist)
+    #         pbihist = pd.DataFrame(pbihist.T)
+    #     n_stop = np.int(np.ceil(stop_sec * self.framessr2))
+    #     pbihist = pbihist.iloc[:np.min([pbihist.shape[0], n_stop]), :]
+    #     print pbihist.shape
+    #     if scale:
+    #         # scale all frames by mean and std of recording
+    #         pbihist = (pbihist - np.nanmean(pbihist)) / np.nanstd(pbihist)
+    #     return pbihist