comparison scripts/classification.py @ 58:d118b6ca8370 branch-tests

some changes in classification with random train/test split

 import map_and_average
 import util_feature_learning
 
 
 FILENAMES = map_and_average.OUTPUT_FILES
-
+TRANSFORM_LABELS = ['LDA', 'PCA', 'NMF', 'SSNMF', 'NA']
 
 def load_data_from_pickle(filename):
     X_list, Y, Yaudio = pickle.load(open(filename,'rb'))
     X = np.concatenate(X_list, axis=1)
     return X, Y, Yaudio

 
 
 def classify_for_filenames(file_list=FILENAMES):
     df_results = pd.DataFrame()
     feat_learner = util_feature_learning.Transformer()
-    for filename in file_list:
+    #traininds, testinds = get_train_test_indices(Yaudio)
+    for filename, transform_label in zip(file_list, TRANSFORM_LABELS):
+        print filename
         X, Y, Yaudio = load_data_from_pickle(filename)
-        traininds, testinds = get_train_test_indices(Yaudio)
-        X_train, Y_train, X_test, Y_test = get_train_test_sets(X, Y, traininds, testinds)
-        df_result = feat_learner.classify(X_train, Y_train, X_test, Y_test)
+        #X_train, Y_train, X_test, Y_test = get_train_test_sets(X, Y, traininds, testinds)
+        X_train, X_val_test, Y_train, Y_val_test = train_test_split(X, Y, train_size=0.6, random_state=12345, stratify=Y)
+        X_val, X_test, Y_val, Y_test = train_test_split(X_val_test, Y_val_test, train_size=0.5, random_state=12345, stratify=Y_val_test)
+        df_result = feat_learner.classify(X_train, Y_train, X_test, Y_test, transform_label=transform_label)
+        df_result_feat = classify_each_feature(X_train, Y_train, X_test, Y_test, transform_label=transform_label)
+        df_result = pd.concat([df_result, df_result_feat], axis=1, ignore_index=True)
         df_results = pd.concat([df_results, df_result], axis=0, ignore_index=True)
     return df_results
 
 
-def classify_each_feature(X_train, Y_train, X_test, Y_test):
+def classify_each_feature(X_train, Y_train, X_test, Y_test, transform_label=" "):
     n_dim = X_train.shape[1]
     feat_labels, feat_inds = map_and_average.get_feat_inds(n_dim=n_dim)
     #df_results = pd.DataFrame()
     # first the classification with all features together
-    df_results = feat_learner.classify(X_train, Y_train, X_test, Y_test)
+    df_results = feat_learner.classify(X_train, Y_train, X_test, Y_test, transform_label=transform_label)
     # then append for each feature separately
     for i in range(len(feat_inds)):
         df_result = feat_learner.classify(X_train[:, feat_inds[i]], Y_train, 
                                           X_test[:, feat_inds[i]], Y_test)
         df_results = pd.concat([df_results, df_result], axis=1, ignore_index=True)

     if saveCF:
         np.savetxt('data/CFlabels.csv', labels, fmt='%s')
         np.savetxt('data/CF.csv', CF, fmt='%10.5f')
     if plots:
         plot_CF(CF, labels=labels, figurename='data/conf_matrix.pdf')
-    return accuracy, predictions
+    return accuracy, CF 
+
+
+def confusion_matrix_for_best_classification_result(df_results, output_data=False):
+    max_i = np.argmax(df_results[:, 1])
+    feat_learning_i = max_i % 4  # 4 classifiers for each feature learning method
+    filename = FILENAMES[feat_learning_i]
+    print filename
+    X, Y, Yaudio = load_data_from_pickle(filename)
+    #traininds, testinds = get_train_test_indices(Yaudio)
+    #X_train, Y_train, X_test, Y_test = get_train_test_sets(X, Y, traininds, testinds)
+    X_train, X_val_test, Y_train, Y_val_test = train_test_split(X, Y, train_size=0.6, random_state=12345, stratify=Y)
+    X_val, X_test, Y_val, Y_test = train_test_split(X_val_test, Y_val_test, train_size=0.5, random_state=12345, stratify=Y_val_test)
+    if output_data:
+        _, CF = confusion_matrix(X_train, Y_train, X_test, Y_test, saveCF=True, plots=True)
+    else:
+        _, CF = confusion_matrix(X_train, Y_train, X_test, Y_test, saveCF=False, plots=False)
+    return CF
 
 
 if __name__ == '__main__':
     df_results = classify_for_filenames(file_list=FILENAMES)
-    max_i = np.argmax(df_results[:, 1])
-    feat_learning_i = max_i % 4  # 4 classifiers for each feature learning method
-    filename = FILENAMES[feat_learning_i]
-    X, Y, Yaudio = load_data_from_pickle(filename)
-    traininds, testinds = get_train_test_indices(Yaudio)
-    X_train, Y_train, X_test, Y_test = get_train_test_sets(X, Y, traininds, testinds)
-    confusion_matrix(X_train, Y_train, X_test, Y_test, saveCF=True, plots=True)
+    CF = confusion_matrix_for_best_classification_result(df_results, output_data=False)