annotate gmm_baseline_experiments/run_experiments.py @ 5:b523456082ca tip

Update path to dataset and reflect modified chunk naming convention.

#!/usr/bin/python

#
# run_experiments.py:
#    Main script for CHiME-Home dataset baseline GMM evaluation
#
# Author: Peter Foster
# (c) 2015 Peter Foster
#

from pylab import *
from sklearn import cross_validation
import os
from pandas import Series, DataFrame
from collections import defaultdict
from extract_features import FeatureExtractor
import exper002
import custompickler
from compute_performance_statistics import compute_performance_statistics
import pdb

Settings = {'paths':{}, 'algorithms':{}}
Settings['paths'] = {'chime_home': {}, 'resultsdir':'/import/c4dm-scratch/peterf/audex/results/', 'featuresdir':'/import/c4dm-scratch/peterf/audex/features/'}
Settings['paths']['chime_home'] = {'basepath':'/import/c4dm-02/people/peterf/audex/datasets/chime_home/release/'}

#Read data sets and class assignments
Datasets = {'chime_home':{}}

#Read in annotations
Chunks = list(Series.from_csv(Settings['paths']['chime_home']['basepath'] + 'chunks_refined.csv',header=None))
Annotations = []
for chunk in Chunks:
    Annotations.append(Series.from_csv(Settings['paths']['chime_home']['basepath'] + 'chunks/' + chunk + '.csv'))
Datasets['chime_home']['dataset'] = DataFrame(Annotations)

#Compute label statistics
Datasets['chime_home']['labelstats'] = defaultdict(lambda: 0)
for item in Datasets['chime_home']['dataset']['majorityvote']:
    for label in item:
        Datasets['chime_home']['labelstats'][label] += 1
#Labels to consider for multilabel classification 
Datasets['chime_home']['consideredlabels'] = ['c', 'b', 'f', 'm', 'o', 'p', 'v']
#Populate binary label assignments
for label in Datasets['chime_home']['consideredlabels']:
    Datasets['chime_home']['dataset'][label] = [label in item for item in Datasets['chime_home']['dataset']['majorityvote']]
#Obtain statistics for considered labels
sum(Datasets['chime_home']['dataset'][Datasets['chime_home']['consideredlabels']]) / len(Datasets['chime_home']['dataset'])
#Create partition for 10-fold cross-validation. Shuffling ensures each fold has approximately equal proportion of label occurrences
np.random.seed(475686)
Datasets['chime_home']['crossval_10fold'] = cross_validation.KFold(len(Datasets['chime_home']['dataset']), 10, shuffle=True)

Datasets['chime_home']['dataset']['wavfile'] = Datasets['chime_home']['dataset']['chunkname'].apply(lambda s: Settings['paths']['chime_home']['basepath'] + 'chunks/' + s + '.48kHz.wav')

#Extract features and assign them to Datasets structure
for dataset in Datasets.keys():
    picklepath = os.path.join(Settings['paths']['featuresdir'],'features_' + dataset)
    if not(os.path.isfile(picklepath)):
        if dataset == 'chime_home':
            featureExtractor = FeatureExtractor(samplingRate=48000, frameLength=1024, hopLength=512)
        else:
            raise NotImplementedError()
        FeatureList = featureExtractor.files_to_features(Datasets[dataset]['dataset']['wavfile'])
        custompickler.pickle_save(FeatureList,picklepath)
    else:
        FeatureList = custompickler.pickle_load(picklepath)
    #Integrity check
    for features in FeatureList:
        for feature in features.values():
            assert(all(isfinite(feature.ravel())))
    Datasets[dataset]['dataset']['features'] = FeatureList
       
#GMM experiments using CHiME home dataset
EXPER005 = {}
EXPER005['name'] = 'GMM_Baseline_EXPER005'
EXPER005['path'] = os.path.join(Settings['paths']['resultsdir'],'exploratory','saved_objects','EXPER005')
EXPER005['settings'] = {'numcomponents': (1,2,4,8), 'features': ('librosa_mfccs',)}
EXPER005['datasets'] = {}
EXPER005['datasets']['chime_home'] = exper002.exper002_multilabelclassification(Datasets['chime_home']['dataset'], Datasets['chime_home']['consideredlabels'], Datasets['chime_home']['crossval_10fold'], Settings, numComponentValues=EXPER005['settings']['numcomponents'], featureTypeValues=EXPER005['settings']['features'])
EXPER005 = compute_performance_statistics(EXPER005, Datasets, Settings, iterableParameters=['numcomponents', 'features'])
custompickler.pickle_save(EXPER005, EXPER005['path'])

#Collate results
def accumulate_results(EXPER):
    EXPER['summaryresults'] = {}
    ds = EXPER['datasets'].keys()[0]
    for numComponents in EXPER['settings']['numcomponents']:
        EXPER['summaryresults'][numComponents] = {}
        for label in Datasets[ds]['consideredlabels']:
            EXPER['summaryresults'][numComponents][label] = EXPER['datasets'][ds][(numComponents, 'librosa_mfccs')]['performance']['classwise'][label]['auc_precisionrecall']
    EXPER['summaryresults'] = DataFrame(EXPER['summaryresults'])
accumulate_results(EXPER005)

#Generate plot
def plot_performance(EXPER):
    fig_width_pt = 246.0 # Get this from LaTeX using \showthe\columnwidth
    inches_per_pt = 1.0/72.27 # Convert pt to inch
    golden_mean = (sqrt(5)-1.0)/2.0 # Aesthetic ratio
    fig_width = fig_width_pt*inches_per_pt # width in inches
    fig_height = fig_width*golden_mean # height in inches
    fig_size =  [fig_width,fig_height]
    params = {'backend': 'ps',
                'axes.labelsize': 8,
                'text.fontsize': 8,
                'legend.fontsize': 7.0,
                'xtick.labelsize': 8,
                'ytick.labelsize': 8,
                'text.usetex': False,
                'figure.figsize': fig_size}
    rcParams.update(params)
    ind = np.arange(len(EXPER['summaryresults'][1]))  # the x locations for the groups
    width = 0.22      # the width of the bars
    fig, ax = plt.subplots()
    rects = []
    colours = ('r', 'y', 'g', 'b', 'c')
    for numComponents, i in zip(EXPER['summaryresults'],range(len(EXPER['summaryresults']))):
        rects.append(ax.bar(ind+width*i, EXPER['summaryresults'][numComponents][['c','m','f','v','p','b','o']], width, color=colours[i], align='center'))
    # add text for labels, title and axes ticks
    ax.set_ylabel('AUC')
    ax.set_xlabel('Label')
    ax.set_xticks(ind+width)
    ax.set_xticklabels(('c','m','f','v','p','b','o'))
    ax.legend( (rect[0] for rect in rects), ('k=1', 'k=2', 'k=4','k=8') ,loc='lower right')
    #Tweak x-axis limit
    ax.set_xlim(left=-0.5)
    ax.set_ylim(top=1.19)
    plt.gcf().subplots_adjust(left=0.15) #Prevent y-axis label from being chopped off
    def autolabel(r):
        for rects in r:
            for rect in rects:
                height = rect.get_height()
                ax.text(rect.get_x()+0.14,0.04+height,'%1.2f'%float(height),ha='center',va='bottom',rotation='vertical',size=6.0)
    autolabel(rects)
    plt.draw()
    plt.savefig('figures/predictionperformance' + EXPER['name'] +'.pdf')
plot_performance(EXPER005)