|
m@95
|
1 import numpy as np
|
|
m@95
|
2 import pandas as pd
|
|
m@95
|
3 import pickle
|
|
m@95
|
4 import os
|
|
m@95
|
5
|
|
m@95
|
6 %load_ext autoreload
|
|
m@95
|
7 %autoreload 2
|
|
m@95
|
8
|
|
m@95
|
9 import sys
|
|
m@95
|
10 sys.path.append('../')
|
|
m@95
|
11 import scripts.outliers as outliers
|
|
m@95
|
12 import scripts.utils as utils
|
|
m@95
|
13
|
|
m@95
|
14 DATA_FILE = '../data/lda_data_melodia_8_30sec.pickle'
|
|
m@95
|
15 METADATA_FILE = '/Users/mariapanteli/Documents/QMUL/Code/MyPythonCode/MergeBL-Smith/data/metadata_BLSM_language_all.csv'
|
|
m@95
|
16 #METADATA_FILE = '../data/metadata.csv'
|
|
m@95
|
17
|
|
m@95
|
18 dataset, ddf, w_dict = outliers.load_data(DATA_FILE, METADATA_FILE)
|
|
m@95
|
19
|
|
m@95
|
20 # correct BL urls:
|
|
m@95
|
21 bl_inds = np.where(np.isnan(ddf['BuyLinkTrackDownload']))[0]
|
|
m@95
|
22 for bl_ind in bl_inds:
|
|
m@95
|
23 ddf['songurls_Album'].iloc[bl_ind] = ('https://sounds.bl.uk/World-and-traditional-music/' +
|
|
m@95
|
24 ddf['Folder'].iloc[bl_ind] + '/' +
|
|
m@95
|
25 ddf['MetaFile'].iloc[bl_ind].split('.')[0])
|
|
m@95
|
26
|
|
m@95
|
27 X_list, Y, Yaudio = dataset
|
|
m@95
|
28 X = np.concatenate(X_list, axis=1)
|
|
m@95
|
29
|
|
m@95
|
30 cols_to_keep = ['Country', 'continent', 'REGION', 'LocDetails',
|
|
m@95
|
31 'Language', 'Language_iso3', 'Culture', 'Genre_Album', 'Year', 'Decade',
|
|
m@95
|
32 'songurls_Album', 'Speech', 'Melspec', 'Chroma', 'Melodia']
|
|
m@95
|
33 cols_rename = ['Country', 'Continent', 'Region', 'Location_details',
|
|
m@95
|
34 'Language', 'Language_iso3', 'Culture', 'Genre', 'Year', 'Decade',
|
|
m@95
|
35 'Url', 'Speech', 'Melspec', 'Chroma', 'Melodia']
|
|
m@95
|
36 col_idx = []
|
|
m@95
|
37 for column in cols_to_keep:
|
|
m@95
|
38 if column not in ddf.columns:
|
|
m@95
|
39 print column
|
|
m@95
|
40 col_idx.append(np.where(ddf.columns==column)[0])
|
|
m@95
|
41 col_idx = np.concatenate(col_idx)
|
|
m@95
|
42 ddf_new = ddf.iloc[:, col_idx]
|
|
m@95
|
43 ddf_new.columns = cols_rename
|
|
m@95
|
44
|
|
m@95
|
45 for i, yy in enumerate(ddf_new['Audio']):
|
|
m@95
|
46 new_name = os.path.split(os.path.split(yy)[0])[-1] + '_' + os.path.split(yy)[-1]
|
|
m@95
|
47 new_csv_name = new_name.split('.')[0]+'.csv'
|
|
m@95
|
48 ddf_new['Audio'].iloc[i] = os.path.join('Audio', new_name)
|
|
m@95
|
49 ddf_new['Chroma'].iloc[i] = os.path.join('Chroma', new_csv_name)
|
|
m@95
|
50 ddf_new['Melspec'].iloc[i] = os.path.join('Melspec', new_csv_name)
|
|
m@95
|
51 ddf_new['Melodia'].iloc[i] = os.path.join('Melodia', new_csv_name)
|
|
m@95
|
52 ddf_new['Speech'].iloc[i] = os.path.join('Speech', new_csv_name)
|
|
m@95
|
53 Yaudio[i] = new_name
|
|
m@95
|
54
|
|
m@95
|
55 ddf_new.to_csv('../data/metadata.csv', index=False)
|
|
m@95
|
56 pickle.dump([X_list, Y, Yaudio], open('../data/lda_data_8.pickle', 'wb'))
|
