comparison scripts/load_dataset.py @ 4:e50c63cf96be branch-tests

rearranging folders

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Mar 15 22:52:57 2017
+
+@author: mariapanteli
+"""
+
+import numpy as np
+import pandas as pd
+import pickle
+
+import load_features
+import util_dataset
+import util_filter_dataset
+
+
+#METADATA_FILE = 'sample_dataset/metadata.csv'
+#OUTPUT_FILES = ['sample_dataset/train_data.pickle', 'sample_dataset/val_data.pickle', 'sample_dataset/test_data.pickle']
+WIN_SIZE = 2
+METADATA_FILE = 'data/metadata_BLSM_language_all.csv'
+#OUTPUT_FILES = ['/import/c4dm-04/mariap/train_data_cf.pickle', '/import/c4dm-04/mariap/val_data_cf.pickle', '/import/c4dm-04/mariap/test_data_cf.pickle']
+#OUTPUT_FILES = ['/import/c4dm-04/mariap/train_data_cf_4.pickle', '/import/c4dm-04/mariap/val_data_cf_4.pickle', '/import/c4dm-04/mariap/test_data_cf_4.pickle']
+OUTPUT_FILES = ['/import/c4dm-04/mariap/train_data_melodia_'+str(WIN_SIZE)+'.pickle', 
+                '/import/c4dm-04/mariap/val_data_melodia_'+str(WIN_SIZE)+'.pickle', 
+                '/import/c4dm-04/mariap/test_data_melodia_'+str(WIN_SIZE)+'.pickle']
+
+def extract_features(df, win2sec=8.0):
+    """Extract features from melspec and chroma.
+    
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Metadata including class label and path to audio, melspec, chroma
+    win2sec : float
+        The window size for the second frame decomposition of the features
+        
+    Returns
+    -------
+    X : np.array
+        The features for every frame x every audio file in the dataset
+    Y : np.array
+        The class labels for every frame in the dataset
+    Y_audio : np.array
+        The audio labels
+    """
+    feat_loader = load_features.FeatureLoader(win2sec=win2sec)
+    frames_rhy, frames_mfcc, frames_chroma, frames_mel, Y_df, Y_audio_df = feat_loader.get_features(df)
+    print frames_rhy.shape, frames_mel.shape, frames_mfcc.shape, frames_chroma.shape
+    X = np.concatenate((frames_rhy, frames_mel, frames_mfcc, frames_chroma), axis=1)
+    Y = Y_df.get_values()
+    Y_audio = Y_audio_df.get_values()
+    return X, Y, Y_audio
+
+
+if __name__ == '__main__':
+    # load dataset
+    df = pd.read_csv(METADATA_FILE)
+    df = util_filter_dataset.remove_missing_data(df)
+    subset_idx = util_dataset.subset_labels(df['Country'].get_values())
+    df = df.iloc[subset_idx, :]
+    X, Y = np.arange(len(df)), df['Country'].get_values()
+    
+    # split in train, val, test set
+    train_set, val_set, test_set = util_dataset.get_train_val_test_idx(X, Y) 
+    
+    # extract features and write output
+    X_train, Y_train, Y_audio_train = extract_features(df.iloc[train_set[0], :], win2sec=WIN_SIZE)
+    with open(OUTPUT_FILES[0], 'wb') as f:
+        pickle.dump([X_train, Y_train, Y_audio_train], f)
+        
+    X_val, Y_val, Y_audio_val = extract_features(df.iloc[val_set[0], :], win2sec=WIN_SIZE)
+    with open(OUTPUT_FILES[1], 'wb') as f:
+        pickle.dump([X_val, Y_val, Y_audio_val], f)
+        
+    X_test, Y_test, Y_audio_test = extract_features(df.iloc[test_set[0], :], win2sec=WIN_SIZE)
+    with open(OUTPUT_FILES[2], 'wb') as f:
+        pickle.dump([X_test, Y_test, Y_audio_test], f)
+
+#out_file = '/import/c4dm-04/mariap/test_data_melodia_1_test.pickle'
+#    pickle.dump([X_test, Y_test, Y_audio_test], f)
+#with open(out_file, 'wb') as f: