Maria@4: # -*- coding: utf-8 -*- Maria@4: """ Maria@4: Created on Tue Jul 12 20:49:48 2016 Maria@4: Maria@4: @author: mariapanteli Maria@4: """ Maria@4: Maria@4: import numpy as np Maria@4: import pandas as pd Maria@4: import pickle Maria@4: from collections import Counter Maria@4: from sklearn.cluster import KMeans Maria@4: Maria@4: import utils Maria@4: import utils_spatial Maria@4: Maria@4: Maria@4: def country_outlier_df(counts, labels, out_file=None, normalize=False): Maria@4: if len(counts.keys()) < len(np.unique(labels)): Maria@4: for label in np.unique(labels): Maria@4: if not counts.has_key(label): Maria@4: counts.update({label:0}) Maria@4: if normalize is True: Maria@4: counts = normalize_outlier_counts(counts, Counter(labels)) Maria@4: df = pd.DataFrame.from_dict(counts, orient='index').reset_index() Maria@4: df.rename(columns={'index':'Country', 0:'Outliers'}, inplace=True) Maria@4: if out_file is not None: Maria@4: df.to_csv(out_file, index=False) Maria@4: return df Maria@4: Maria@4: Maria@4: def normalize_outlier_counts(outlier_counts, country_counts): Maria@4: '''Normalize a dictionary of outlier counts per country by Maria@4: the total number of recordings per country Maria@4: ''' Maria@4: for key in outlier_counts.keys(): Maria@4: # dictionaries should have the same keys Maria@4: outlier_counts[key] = float(outlier_counts[key]) / float(country_counts[key]) Maria@4: return outlier_counts Maria@4: Maria@4: Maria@4: def get_outliers_df(X, Y, chi2thr=0.999, out_file=None): Maria@4: threshold, y_pred, MD = utils.get_outliers_Mahal(X, chi2thr=chi2thr) Maria@4: global_counts = Counter(Y[y_pred]) Maria@4: df = country_outlier_df(global_counts, Y, normalize=True) Maria@4: if out_file is not None: Maria@4: df.to_csv(out_file, index=False) Maria@4: return df, threshold, MD Maria@4: Maria@4: Maria@4: def print_most_least_outliers_topN(df, N=10): Maria@4: sort_inds = df['Outliers'].argsort() # ascending order Maria@4: df_most = df[['Country', 'Outliers']].iloc[sort_inds[::-1][:N]] Maria@4: df_least = df[['Country', 'Outliers']].iloc[sort_inds[:N]] Maria@4: print "most outliers " Maria@4: print df_most Maria@4: print "least outliers " Maria@4: print df_least Maria@4: Maria@4: Maria@4: def load_metadata(Yaudio, metadata_file): Maria@4: df = pd.read_csv(metadata_file) Maria@4: df_audio = pd.DataFrame({'Audio':Yaudio}) Maria@4: ddf = pd.merge(df_audio, df, on='Audio', suffixes=['', '_r']) # in the order of Yaudio Maria@4: return ddf Maria@4: Maria@4: Maria@4: def clusters_metadata(df, cl_pred, out_file=None): Maria@4: def get_top_N_counts(labels, N=3): Maria@4: ulab, ucount = np.unique(labels, return_counts=True) Maria@4: inds = np.argsort(ucount) Maria@4: return zip(ulab[inds[-N:]],ucount[inds[-N:]]) Maria@4: info = np.array([str(df['Country'].iloc[i]) for i in range(len(df))]) Maria@4: styles_description = [] Maria@4: uniq_cl = np.unique(cl_pred) Maria@4: for ccl in uniq_cl: Maria@4: inds = np.where(cl_pred==ccl)[0] Maria@4: styles_description.append(get_top_N_counts(info[inds], N=3)) Maria@4: df_styles = pd.DataFrame(data=styles_description, index=uniq_cl) Maria@4: print df_styles.to_latex() Maria@4: if out_file is not None: Maria@4: df_styles.to_csv(out_file, index=False) Maria@4: Maria@4: m@8: if __name__ == '__main__': m@8: # load LDA-transformed frames m@8: X_list, Y, Yaudio = pickle.load(open('data/lda_data_8.pickle','rb')) m@8: ddf = load_metadata(Yaudio, metadata_file='data/metadata.csv') m@8: w, data_countries = utils_spatial.get_neighbors_for_countries_in_dataset(Y) m@8: w_dict = utils_spatial.from_weights_to_dict(w, data_countries) m@13: X = np.concatenate(X_list, axis=1) Maria@4: m@8: # global outliers m@8: df_global, threshold, MD = get_outliers_df(X, Y, chi2thr=0.999) m@8: print_most_least_outliers_topN(df_global, N=10) Maria@4: m@8: spatial_outliers = utils.get_local_outliers_from_neighbors_dict(X, Y, w_dict, chi2thr=0.999, do_pca=True) m@8: spatial_counts = Counter(dict([(ll[0],ll[1]) for ll in spatial_outliers])) m@8: df_local = country_outlier_df(spatial_counts, Y, normalize=True) m@8: print_most_least_outliers_topN(df_local, N=10) Maria@4: m@8: feat = [Xrhy, Xmel, Xmfc, Xchr] m@8: feat_labels = ['rhy', 'mel', 'mfc', 'chr'] m@8: tabs_feat = [] m@8: for i in range(len(feat)): m@8: XX = feat[i] m@8: df_feat, threshold, MD = get_outliers_df(XX, Y, chi2thr=0.999) m@8: print_most_least_outliers_topN(df_feat, N=5) Maria@4: m@8: # how many styles are there m@8: #bestncl, ave_silh = utils.best_n_clusters_silhouette(X, min_ncl=5, max_ncl=50, metric="cosine") m@8: bestncl = 13 Maria@4: m@8: # get cluster predictions and metadata for each cluster m@8: cluster_model = KMeans(n_clusters=bestncl, random_state=50).fit(X) m@8: centroids = cluster_model.cluster_centers_ m@8: cl_pred = cluster_model.predict(X) m@8: ddf['Clusters'] = cl_pred m@8: clusters_metadata(ddf, cl_pred) Maria@4: m@8: # how similar are the cultures and which ones seem to be global outliers m@8: cluster_freq = utils.get_cluster_freq_linear(X, Y, centroids) Maria@4: m@8: # Moran on Mahalanobis distances m@8: data = cluster_freq.get_values() m@8: data_countries = cluster_freq.index m@8: #threshold, y_pred, MD = utils.get_outliers_Mahal(data, chi2thr=0.999) m@8: threshold, y_pred, MD = utils.get_outliers(data, chi2thr=0.999) m@8: y = np.sqrt(MD) m@8: utils_spatial.print_Moran_outliers(y, w, data_countries) m@8: utils_spatial.plot_Moran_scatterplot(y, w, data_countries)