tomwalters@54: #!/bin/bash
tomwalters@54: # Copyright 2010 Thomas Walters <tom@acousticscale.org>
tomwalters@54: #
tomwalters@54: # Run a series of experiments which compare MFCC features generated by HTK to
tomwalters@54: # AIM features generated using AIM-C using a series of syllable recogntiton
tomwalters@54: # tasks.
tomwalters@165: # This script expects the HTK binaries and AIM-C AIMCopy binary to be present
tomwalters@165: # in the PATH.
tomwalters@128: 
tomwalters@54: # Set these to be the location of your input database, and desired output
tomwalters@165: # locations.
tomwalters@178: SYLLABLES_DATABASE_TAR=/mnt/sounds/cnbh-syllables.tar
tomwalters@165: SOUNDS_ROOT=/mnt/experiments/sounds/
tomwalters@165: FEATURES_ROOT=/mnt/experiments/features/
tomwalters@165: HMMS_ROOT=/mnt/experiments/hmms/
tomwalters@128: 
tomwalters@54: # Number of cores on the experimental machine. Various scripts will try to use
tomwalters@54: # this if it's set.
tomwalters@184: NUMBER_OF_CORES=1
tomwalters@54: 
tomwalters@54: # Fail if any command fails
tomwalters@54: set -e
tomwalters@54: 
tomwalters@54: # Fail if any variable is unset
tomwalters@54: set -u
tomwalters@54: 
tomwalters@178: if [ ! -e $SYLLABLES_DATABASE_TAR ]; then
tomwalters@178:   sudo mkdir -p `dirname $SYLLABLES_DATABASE_TAR`
tomwalters@178:   sudo chown ubuntu `dirname $SYLLABLES_DATABASE_TAR`
tomwalters@178:   wget -O $SYLLABLES_DATABASE_TAR $SYLLABLES_DATABASE_URL
tomwalters@178: fi
tomwalters@178: 
tomwalters@54: if [ ! -d $SOUNDS_ROOT ]; then
tomwalters@169:   sudo mkdir -p $SOUNDS_ROOT
tomwalters@169:   sudo chown `whoami` $SOUNDS_ROOT
tomwalters@54: fi
tomwalters@54: 
tomwalters@165: # Untar the CNBH syllables database, and convert the files from FLAC to WAV
tomwalters@54: if [ ! -e $SOUNDS_ROOT/.untar_db_success ]; then
tomwalters@54:   tar -x -C $SOUNDS_ROOT -f $SYLLABLES_DATABASE_TAR
tomwalters@54:   touch $SOUNDS_ROOT/.untar_db_success
tomwalters@54: fi
tomwalters@54: 
tomwalters@54: # Convert the database to .WAV format and place it in $SOUNDS_ROOT/clean
tomwalters@185: echo "Converting CNBH-syllables database from FLAC to WAV..."
tomwalters@171: ./cnbh-syllables/feature_generation/convert_flac_to_wav.sh $SOUNDS_ROOT
tomwalters@54: 
tomwalters@54: # Generate versions of the CNBH syllables spoke pattern with a range of
tomwalters@54: # signal-to-noise ratios (SNRs). The versions are put in the directory
tomwalters@54: # ${SOUNDS_ROOT}/${SNR}_dB/ for each SNR in $SNRS.
tomwalters@191: #SNRS="30 27 24 21 18 15 12 9 6 3 0"
tomwalters@191: SNRS="30" # For testing
tomwalters@186: ./cnbh-syllables/feature_generation/pink_noise.sh $SOUNDS_ROOT/clean/ "$SNRS"
tomwalters@54: 
tomwalters@54: # Make the list of all feature drectories
tomwalters@165: FEATURE_DIRS="clean"
tomwalters@54: for SNR in $SNRS; do
tomwalters@54:   FEATURE_DIRS="$FEATURE_DIRS snr_${SNR}dB"
tomwalters@54: done
tomwalters@54: 
tomwalters@54: # Generate feature sets (for the full range of SNRs in $FEATURE_DIRS)
tomwalters@54: # 1. Standard MFCC features
tomwalters@54: # 2. AIM features
tomwalters@165: # 3. MFCC features with optimal VTLN 
tomwalters@169: 
tomwalters@169: 
tomwalters@169: if [ ! -d $FEATURES_ROOT ]; then
tomwalters@169:   sudo mkdir -p $FEATURES_ROOT
tomwalters@169:   sudo chown `whoami` $FEATURES_ROOT
tomwalters@169: fi
tomwalters@169: 
tomwalters@180: if [ ! -e /mnt/experiments/htk/.htk_installed_success ]; then
tomwalters@180:   ./HTK/install_htk.sh
tomwalters@181: fi
tomwalters@180: 
tomwalters@184: if [ ! -e /mnt/experiments/aimc/.aimc_build_success ]; then
tomwalters@183: # ./aimc/build_aimc.sh
tomwalters@183:   cd ../../
tomwalters@183:   scons
tomwalters@183:   export PATH=$PATH:`pwd`/build/posix-release/
tomwalters@183:   cd -
tomwalters@183: fi
tomwalters@183: 
tomwalters@54: for SOURCE_SNR in $FEATURE_DIRS; do
tomwalters@165:   
tomwalters@177:   if [ ! -e $FEATURES_ROOT/mfcc/$SOURCE_SNR/.make_mfcc_features_success ]; then
tomwalters@54:     mkdir -p $FEATURES_ROOT/mfcc/$SOURCE_SNR/
tomwalters@54:     # Generate the list of files to convert
tomwalters@189:     ./cnbh-syllables/feature_generation/gen_hcopy_aimcopy_script.sh $FEATURES_ROOT/mfcc/$SOURCE_SNR/ $SOUNDS_ROOT/$SOURCE_SNR/ htk
tomwalters@54:     # Run the conversion
tomwalters@101:     ./cnbh-syllables/feature_generation/run_hcopy.sh $FEATURES_ROOT/mfcc/$SOURCE_SNR/ $NUMBER_OF_CORES
tomwalters@101:     touch $FEATURES_ROOT/mfcc/$SOURCE_SNR/.make_mfcc_features_success
tomwalters@176:   fi
tomwalters@54: 
tomwalters@177:   if [ ! -e $FEATURES_ROOT/mfcc_vtln/$SOURCE_SNR/.make_mfcc_vtln_features_success ]; then
tomwalters@54:     mkdir -p $FEATURES_ROOT/mfcc_vtln/$SOURCE_SNR/
tomwalters@54:     # Generate the file list and run the conversion (all one step, since this
tomwalters@186:     # version uses a different configuration for each talker)
tomwalters@101:     ./cnbh-syllables/feature_generation/run_mfcc_vtln_conversion.sh $FEATURES_ROOT/mfcc_vtln/$SOURCE_SNR/ $SOUNDS_ROOT/$SOURCE_SNR/
tomwalters@101:     touch $FEATURES_ROOT/mfcc_vtln/$SOURCE_SNR/.make_mfcc_vtln_features_success
tomwalters@176:   fi
tomwalters@54: 
tomwalters@177:   if [ ! -e $FEATURES_ROOT/aim/$SOURCE_SNR/.make_aim_features_success ]; then
tomwalters@165:     mkdir -p $FEATURES_ROOT/aim/$SOURCE_SNR/ 
tomwalters@189:     ./cnbh-syllables/feature_generation/gen_hcopy_aimcopy_script.sh $FEATURES_ROOT/aim/$SOURCE_SNR/ $SOUNDS_ROOT/$SOURCE_SNR/ ""
tomwalters@54:     # Run the conversion
tomwalters@191:     #./cnbh-syllables/feature_generation/run_aimcopy.sh $FEATURES_ROOT/aim/$SOURCE_SNR/ $NUMBER_OF_CORES
tomwalters@191:     #touch $FEATURES_ROOT/aim/$SOURCE_SNR/.make_aim_features_success
tomwalters@176:   fi
tomwalters@165: done 
tomwalters@82: 
tomwalters@193: sudo mkdir -p $HMMS_ROOT
tomwalters@193: sudo chown ubuntu $HMMS_ROOT
tomwalters@193: 
tomwalters@54: # Now run a bunch of experiments.
tomwalters@54: # For each of the feature types, we want to run HMMs with a bunch of
tomwalters@54: # parameters.
tomwalters@165: TRAINING_ITERATIONS="0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20"
tomwalters@165: TESTING_ITERATIONS="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20"
tomwalters@165: HMM_STATES="3 4 5 6 7 8"
tomwalters@187: HMM_OUTPUT_COMPONENTS="1 2 3 4 5 6"
tomwalters@91: 
tomwalters@186: FEATURE_CLASS=mfcc
tomwalters@190: FEATURE_SUFFIX=htk
tomwalters@186: FEATURE_SIZE=39
tomwalters@186: FEATURE_TYPE=MFCC_0_D_A
tomwalters@93: 
tomwalters@189: TALKERS=inner_talkers
tomwalters@189: WORK=$HMMS_ROOT/$FEATURE_CLASS/$FEATURE_SUFFIX/$SOURCE_SNR/$TALKERS/
tomwalters@193: mkdir -p $WORK
tomwalters@189: FEATURES_DIR=$FEATURES_ROOT/$FEATURE_CLASS/$SOURCE_SNR/
tomwalters@189: 
tomwalters@191: ./cnbh-syllables/run_training_and_testing/train_test_sets/generate_train_test_lists.sh \
tomwalters@187:     $TALKERS \
tomwalters@187:     $WORK \
tomwalters@187:     $FEATURES_DIR \
tomwalters@187:     $FEATURE_SUFFIX
tomwalters@187: 
tomwalters@192: TRAINING_SCRIPT=$WORK/training_script
tomwalters@192: TESTING_SCRIPT=$WORK/testing_script
tomwalters@192: 
tomwalters@186: for SOURCE_SNR in $FEATURE_DIRS; do
tomwalters@191:   ./cnbh-syllables/run_training_and_testing/test_features.sh \
tomwalters@186:       $HMMS_ROOT \
tomwalters@186:       $FEATURES_ROOT/$FEATURE_CLASS/$SOURCE_SNR/ \
tomwalters@186:       $FEATURE_SUFFIX \
tomwalters@186:       $HMM_STATES \
tomwalters@186:       $HMM_OUTPUT_COMPONENTS \
tomwalters@186:       $TRAINING_ITERATIONS \
tomwalters@186:       $TESTING_ITERATIONS \
tomwalters@186:       $FEATURE_SIZE \
tomwalters@187:       $FEATURE_TYPE \
tomwalters@187:       $TRAINING_SCRIPT \
tomwalters@187:       $TESTING_SCRIPT
tomwalters@192: done