comparison SegEval.py @ 4:56a2ca9359d0

evaluation script

 import novelty as novelty_S
 
 # Algorithm params
 h = 8               # Size of median filter for features in C-NMF
 R = 15              # Size of the median filter for the activation matrix C-NMF
-rank = 8            # Rank of decomposition for the boundaries
+rank = 4            # Rank of decomposition for the boundaries
 rank_labels = 6     # Rank of decomposition for the labels
 R_labels = 6        # Size of the median filter for the labels
 # Foote
 M = 2           # Median filter for the audio features (in beats)
 Mg = 32         # Gaussian kernel size

 		'''Settings for extracting tempogram features.'''
 		self.tempoWindow = 6.0
 		self.bpmBands = [30, 45, 60, 80, 100, 120, 180, 240, 400, 600]
 		
 		'''Peak picking settings for novelty based method'''
-		self.threshold = 50
+		self.threshold = 30
 		self.confidence_threshold = 0.5
 		self.delta_threshold = 0.0
 		self.backtracking_threshold = 1.9
 		self.polyfitting_on = True
 		self.medfilter_on = True

 				
 			
 	def pairwiseF(self, annotation, detection, tolerance=3.0, combine=1.0, idx2time=None):
 		'''Pairwise F measure evaluation of detection rates.'''
 		
-		if idx2time:
+		if idx2time != None:
 			# Map detected idxs to real time
-			detection = [idx2time[int(np.rint(i))] for i in detection] + [ao.gt[-1]]
+			detection = [idx2time[int(np.rint(i))] for i in detection] + [annotation[-1]]
 		# print 'detection', detection
 		detection = np.append(detection, annotation[-1])
 		res = EvalObj()
 		res.TP = 0	# Total number of matched ground truth and experimental data points
 		gt = len(annotation)	# Total number of ground truth data points

 		res.P = res.TP / float(dt)
 		res.R = res.TP / float(gt)
 		res.F = 2 * res.P * res.R / (res.P + res.R)
 		return res
 		
-	def wirteIndividualHeader(self, filename):
+	def writeIndividualHeader(self, filename):
 		'''Write header of output files for individual features.'''
 		
 		with open(filename, 'a') as f:
 			csvwriter = csv.writer(f, delimiter=',')
 			csvwriter.writerow(['audio', 'gammatone_tp_05', 'gammatone_fp_05', 'gammatone_fn_05', 'gammatone_P_05', 'gammatone_R_05', 'gammatone_F_05', 'gammatone_AD_05', 'gammatone_DA_05', 'gammatone_tp_3', \

 			'harmonic_R_05', 'harmonic_F_05', 'harmonic_AD_05', 'harmonic_DA_05', 'harmonic_tp_3', 'harmonic_fp_3', 'harmonic_fn_3', 'harmonic_P_3', 'harmonic_R_3', 'harmonic_F_3', 'harmonic_AD_3', 'harmonic_DA_3', \
 			'timbre_tp_05', 'timbre_fp_05', 'timbre_fn_05', 'timbre_P_05', 'timbre_R_05', 'timbre_F_05', 'timbre_AD_05', 'timbre_DA_05', 'timbre_tp_3', 'timbre_fp_3', 'timbre_fn_3', 'timbre_P_3', 'timbre_R_3', \
 			'timbre_F_3', 'timbre_AD_3', 'timbre_DA_3',	 'tempo_tp_05', 'tempo_fp_05', 'tempo_fn_05', 'tempo_P_05', 'tempo_R_05', 'tempo_F_05', 'tempo_AD_05', 'tempo_DA_05', \
 			'tempo_tp_3', 'tempo_fp_3', 'tempo_fn_3', 'tempo_P_3', 'tempo_R_3', 'tempo_F_3', 'tempo_AD_3', 'tempo_DA_3'])
 	
-	def wirteIndividualRes(self, filename, ao_name, gt_res_05, gt_res_3, harmonic_res_05, harmonic_res_3, timbre_res_05, timbre_res_3, tempo_res_05, tempo_res_3):
+	def writeIndividualRes(self, filename, ao_name, gt_res_05, gt_res_3, harmonic_res_05, harmonic_res_3, timbre_res_05, timbre_res_3, tempo_res_05, tempo_res_3):
 		'''Write result of single detection for individual features.'''
 		
 		with open(filename, 'a') as f:
 			csvwriter = csv.writer(f, delimiter=',')
 			csvwriter.writerow([ao_name, gt_res_05.TP, gt_res_05.FP, gt_res_05.FN, gt_res_05.P, gt_res_05.R, gt_res_05.F, gt_res_05.AD, gt_res_05.DA, gt_res_3.TP, gt_res_3.FP, gt_res_3.FN, gt_res_3.P, \

 		'''Write header of output files for combined features.'''
 		
 		with open(filename, 'a') as f:
 			csvwriter = csv.writer(f, delimiter=',')
 			csvwriter.writerow(['audio', 'gt_tb_P_0.5', 'gt_tb_R_0.5', 'gt_tb_F_0.5', 'gt_tb_P_3', 'gt_tb_R_3', 'gt_tb_F_3', 'gt_tp_P_0.5', 'gt_tp_R_0.5', 'gt_tp_F_0.5', 'gt_tp_P_3', 'gt_tp_R_3', 'gt_tp_F_3',\
-			'gt_hm_P_0.5', 'gt_hm_R_0.5', 'gt_hm_F_0.5', 'gt_hm_P_3', 'gt_hm_R_3', 'gt_hm_F_3', 'tb_tp_P_0.5', 'tb_tp_R_0.5', 'tb_tp_F_0.5', 'tb_tp_P_3', 'tb_tp_R_3', 'tb_tp_F_3', 'tb_hm_P_0.5', 'tb_hm_R_0.5', 'tb_hm_F_0.5', \
-			'tb_hm_P_3', 'tb_hm_R_3', 'tb_hm_F_3', 'tp_hm_P_0.5', 'tp_hm_R_0.5', 'tp_hm_F_0.5', 'tp_hm_P_3', 'tp_hm_R_3', 'tp_hm_F_3', 'gt_tb_tp_P_0.5', 'gt_tb_tp_R_0.5', 'gt_tb_tp_F_0.5', 'gt_tb_tp_P_3', 'gt_tb_tp_R_3', \
-			'gt_tb_tp_F_3', 'gt_tb_hm_P_0.5', 'gt_tb_hm_R_0.5', 'gt_tb_hm_F_0.5', 'gt_tb_hm_P_3', 'gt_tb_hm_R_3', 'gt_tb_hm_F_3', 'gt_tp_hm_P_0.5', 'gt_tp_hm_R_0.5', 'gt_tp_hm_F_0.5', 'gt_tp_hm_P_3', 'gt_tp_hm_R_3', 'gt_tp_hm_F_3', \
-			'tb_tp_hm_P_0.5', 'tb_tp_hm_R_0.5', 'tb_tp_hm_F_0.5', 'tb_tp_hm_P_3', 'tb_tp_hm_R_3', 'tb_tp_hm_F_3', 'gt_tb_tp_hm_P_0.5', 'gt_tb_tp_hm_R_0.5', 'gt_tb_tp_hm_F_0.5', 'gt_tb_tp_hm_P_3', 'gt_tb_tp_hm_R_3', 'gt_tb_tp_hm_F_3'])
-		
-	def wirteCombinedRes(self, filename, ao_name, gt_hm_res_05, gt_hm_res_3, gt_tb_res_05, gt_tb_res_3, gt_tp_res_05, gt_tp_res_3, hm_tb_res_05, hm_tb_res_3, hm_tp_res_05, hm_tp_res_3, \
-	tb_tp_res_05, tb_tp_res_3, gt_hm_tb_res_05, gt_hm_tb_res_3, gt_hm_tp_res_05, gt_hm_tp_res_3, gt_tb_tp_res_05, gt_tb_tp_res_3, hm_tb_tp_res_05, hm_tb_tp_res_3):
+			'gt_hm_P_0.5', 'gt_hm_R_0.5', 'gt_hm_F_0.5', 'gt_hm_P_3', 'gt_hm_R_3', 'gt_hm_F_3', 'tb_tp_P_0.5', 'tb_tp_R_0.5', 'tb_tp_F_0.5', 'tb_tp_P_3', 'tb_tp_R_3', 'tb_tp_F_3', 'hm_tb_P_0.5', 'hm_tb_R_0.5', 'hm_tb_F_0.5', \
+			'hm_tb_P_3', 'hm_tb_R_3', 'hm_tb_F_3', 'hm_tp_P_0.5', 'hm_tp_R_0.5', 'hm_tp_F_0.5', 'hm_tp_P_3', 'hm_tp_R_3', 'hm_tp_F_3', 'gt_tb_tp_P_0.5', 'gt_tb_tp_R_0.5', 'gt_tb_tp_F_0.5', 'gt_tb_tp_P_3', 'gt_tb_tp_R_3', \
+			'gt_tb_tp_F_3', 'gt_hm_tb_P_0.5', 'gt_hm_tb_R_0.5', 'gt_hm_tb_F_0.5', 'gt_hm_tb_P_3', 'gt_hm_tb_R_3', 'gt_hm_tb_F_3', 'gt_hm_tp_P_0.5', 'gt_hm_tp_R_0.5', 'gt_hm_tp_F_0.5', 'gt_hm_tp_P_3', 'gt_hm_tp_R_3', 'gt_hm_tp_F_3', \
+			'hm_tb_tp_P_0.5', 'hm_tb_tp_R_0.5', 'hm_tb_tp_F_0.5', 'hm_tb_tp_P_3', 'hm_tb_tp_R_3', 'hm_tb_tp_F_3', 'gt_hm_tb_tp_P_0.5', 'gt_hm_tb_tp_R_0.5', 'gt_hm_tb_tp_F_0.5', 'gt_hm_tb_tp_P_3', 'gt_hm_tb_tp_R_3', 'gt_hm_tb_tp_F_3'])
+		
+	def writeCombinedRes(self, filename, ao_name, gt_hm_res_05, gt_hm_res_3, gt_tb_res_05, gt_tb_res_3, gt_tp_res_05, gt_tp_res_3, hm_tb_res_05, hm_tb_res_3, hm_tp_res_05, hm_tp_res_3, \
+	tb_tp_res_05, tb_tp_res_3, gt_hm_tb_res_05, gt_hm_tb_res_3, gt_hm_tp_res_05, gt_hm_tp_res_3, gt_tb_tp_res_05, gt_tb_tp_res_3, hm_tb_tp_res_05, hm_tb_tp_res_3, gt_hm_tb_tp_res_05, gt_hm_tb_tp_res_3):
 		'''Write result of single detection for combined features.'''
 		
 		with open(filename, 'a') as f:
 			csvwriter = csv.writer(f, delimiter=',')
 			csvwriter.writerow([ao_name, gt_tb_res_05.P, gt_tb_res_05.R, gt_tb_res_05.F, gt_tb_res_3.P, gt_tb_res_3.R, gt_tb_res_3.F, gt_tp_res_05.P, gt_tp_res_05.R, gt_tp_res_05.F, gt_tp_res_3.P, gt_tp_res_3.R, gt_tp_res_3.F, \
 			gt_hm_res_05.P, gt_hm_res_05.R, gt_hm_res_05.F, gt_hm_res_3.P, gt_hm_res_3.R, gt_hm_res_3.F, tb_tp_res_05.P, tb_tp_res_05.R, tb_tp_res_05.F, tb_tp_res_3.P, tb_tp_res_3.R, tb_tp_res_3.F, \
-			tb_hm_res_05.P, tb_hm_res_05.R, tb_hm_res_05.F, tb_hm_res_3.P, tb_hm_res_3.R, tb_hm_res_3.F, tp_hm_res_05.P, tp_hm_res_05.R, tp_hm_res_05.F, tp_hm_res_3.P, tp_hm_res_3.R, tp_hm_res_3.F, \
-			gt_tb_tp_res_05.P, gt_tb_tp_res_05.R, gt_tb_tp_res_05.F, gt_tb_tp_res_3.P, gt_tb_tp_res_3.R, gt_tb_tp_res_3.F, gt_tb_hm_res_05.P, gt_tb_hm_res_05.R, gt_tb_hm_res_05.F, gt_tb_hm_res_3.P, gt_tb_hm_res_3.R, gt_tb_hm_res_3.F, \
-			gt_tp_hm_res_05.P, gt_tp_hm_res_05.R, gt_tp_hm_res_05.F, gt_tp_hm_res_3.P, gt_tp_hm_res_3.R, gt_tp_hm_res_3.F, tb_tp_hm_res_05.P, tb_tp_hm_res_05.R, tb_tp_hm_res_05.F, tb_tp_hm_res_3.P, tb_tp_hm_res_3.R, tb_tp_hm_res_3.F, \
-			gt_tb_tp_hm_res_05.P, gt_tb_tp_hm_res_05.R, gt_tb_tp_hm_res_05.F, gt_tb_tp_hm_res_3.P, gt_tb_tp_hm_res_3.R, gt_tb_tp_hm_res_3.F])
-		
-		
+			hm_tb_res_05.P, hm_tb_res_05.R, hm_tb_res_05.F, hm_tb_res_3.P, hm_tb_res_3.R, hm_tb_res_3.F, hm_tp_res_05.P, hm_tp_res_05.R, hm_tp_res_05.F, hm_tp_res_3.P, hm_tp_res_3.R, hm_tp_res_3.F, \
+			gt_tb_tp_res_05.P, gt_tb_tp_res_05.R, gt_tb_tp_res_05.F, gt_tb_tp_res_3.P, gt_tb_tp_res_3.R, gt_tb_tp_res_3.F, gt_hm_tb_res_05.P, gt_hm_tb_res_05.R, gt_hm_tb_res_05.F, gt_hm_tb_res_3.P, gt_hm_tb_res_3.R, gt_hm_tb_res_3.F, \
+			gt_hm_tp_res_05.P, gt_hm_tp_res_05.R, gt_hm_tp_res_05.F, gt_hm_tp_res_3.P, gt_hm_tp_res_3.R, gt_hm_tp_res_3.F, hm_tb_tp_res_05.P, hm_tb_tp_res_05.R, hm_tb_tp_res_05.F, hm_tb_tp_res_3.P, hm_tb_tp_res_3.R, hm_tb_tp_res_3.F, \
+			gt_hm_tb_tp_res_05.P, gt_hm_tb_tp_res_05.R, gt_hm_tb_tp_res_05.F, gt_hm_tb_tp_res_3.P, gt_hm_tb_tp_res_3.R, gt_hm_tb_tp_res_3.F])
+		
+	def writeMergedHeader(self, filename):
+		'''Write header of output files merging individual detections.'''
+		with open(filename, 'a') as f:
+			csvwriter = csv.writer(f, delimiter=',')
+			csvwriter.writerow(['audio', 'merged_tp_05', 'merged_fp_05', 'merged_fn_05', 'merged_P_05', 'merged_R_05', 'merged_F_05', 'merged_AD_05', 'merged_DA_05', 'merged_tp_3', \
+			'merged_fp_3', 'merged_fn_3', 'merged_P_3', 'merged_R_3', 'merged_F_3', 'merged_AD_3', 'merged_DA_3'])
+			
+	def writeMergedRes(self, filename, ao_name, merged_res_05, merged_res_3):
+		'''Write results by merging individual detections.'''
+		with open(filename, 'a') as f:
+			csvwriter = csv.writer(f, delimiter=',')
+			csvwriter.writerow([ao_name, merged_res_05.TP, merged_res_05.FP, merged_res_05.FN, merged_res_05.P, merged_res_05.R, merged_res_05.F, merged_res_05.AD, merged_res_05.DA, \
+			merged_res_3.TP, merged_res_3.FP, merged_res_3.FN, merged_res_3.P, merged_res_3.R, merged_res_3.F, merged_res_3.AD, merged_res_3.DA])
+			
+			
 	def process(self):
 		'''For the aggregated input features, discard a propertion each time as the pairwise distances within the feature space descending. 
 		In the meanwhile evaluate the segmentation result and track the trend of perfomance changing by measuring the feature selection 
 		threshold - segmentation f measure curve. 
 		'''

 		feature_step_frame = int(0.5 * self.SampleRate / self.gammatoneLen * self.featureStep)
 		aggregation_window, aggregation_step = 100, 50
 		featureRate = float(self.SampleRate) / self.stepSize
 		
 		audio_files = [x for x in os.listdir(options.GT) if not x.startswith(".") ]
-		# audio_files = audio_files[:2]
+		if options.TEST:
+			audio_files = audio_files[:1]
 		audio_files.sort()
 		audio_list = []
 
 		gammatone_feature_list = [i for i in os.listdir(options.GF) if not i.startswith('.')]
-		gammatone_feature_list = ['contrast4', 'rolloff', 'dct']
+		gammatone_feature_list = ['contrast6', 'rolloff4', 'dct']
 		tempo_feature_list = [i for i in os.listdir(options.TF) if not i.startswith('.')]
-		tempo_feature_list = ['intensity_bpm', 'loudness_bpm']
-		timbre_feature_list = ['mfcc']
-		harmonic_feature_list = ['nnls']
+		tempo_feature_list = ['ti', 'tir']
+		timbre_feature_list = ['mfcc_harmonic']
+		harmonic_feature_list = ['chromagram']
 
 		gammatone_feature_list = [join(options.GF, f) for f in gammatone_feature_list]
 		timbre_feature_list = [join(options.SF, f) for f in timbre_feature_list]
 		tempo_feature_list = [join(options.TF, f) for f in tempo_feature_list]
 		harmonic_feature_list = [join(options.SF, f) for f in harmonic_feature_list]

 
 		# For each audio file, load specific features
 		for audio in audio_files:
 			ao = AudioObj()
 			ao.name = splitext(audio)[0]
-			print ao.name
 			# annotation_file = join(options.GT, ao.name+'.txt') # iso, salami
 			# ao.gt = np.genfromtxt(annotation_file, usecols=0)	
 			# ao.label = np.genfromtxt(annotation_file, usecols=1, dtype=str)
 			annotation_file = join(options.GT, ao.name+'.csv') # qupujicheng
 			ao.gt = np.genfromtxt(annotation_file, usecols=0, delimiter=',')	

 			aggregation_window, aggregation_step = 1,1
 			featureRate = float(self.SampleRate) /self.stepSize
 			pca = PCA(n_components=5)
 			
 			# Resample and normalise features
+			step = ao.tempo_features.shape[0]
 			ao.gammatone_features = resample(ao.gammatone_features, step)
 			ao.gammatone_features = normaliseFeature(ao.gammatone_features)
 			ao.timbre_features = resample(ao.timbre_features, step)
 			ao.timbre_features = normaliseFeature(ao.timbre_features)
 			ao.harmonic_features = resample(ao.harmonic_features, step)
 			ao.harmonic_features = normaliseFeature(ao.harmonic_features)
-			ao.tempo_features = normaliseFeature(ao.harmonic_features)
+			ao.tempo_features = normaliseFeature(ao.tempo_features)
 			
 			pca.fit(ao.gammatone_features)
 			ao.gammatone_features = pca.transform(ao.gammatone_features)
 			ao.gammatone_ssm = getSSM(ao.gammatone_features)
 			

 		outfile5 = join(options.OUTPUT, 'combined_novelty.csv')
 		outfile6 = join(options.OUTPUT, 'combined_foote.csv')
 		outfile7 = join(options.OUTPUT, 'combined_sf.csv')
 		outfile8 = join(options.OUTPUT, 'combined_cnmf.csv')
 		
-		self.wirteIndividualHeader(outfile1)
-		self.wirteIndividualHeader(outfile2)
-		self.wirteIndividualHeader(outfile3)
-		self.wirteIndividualHeader(outfile4)
-			
-		# self.wirteCombinedlHeader(outfile5)
-		# self.wirteCombinedlHeader(outfile6)
-		self.wirteCombinedlHeader(outfile7)
-		# self.wirteCombinedlHeader(outfile8)
+		outfile9 = join(options.OUTPUT, 'individual_merged.csv')
+		
+		self.writeIndividualHeader(outfile1)
+		self.writeIndividualHeader(outfile2)
+		self.writeIndividualHeader(outfile3)
+		self.writeIndividualHeader(outfile4)
+			
+		# self.writeCombinedHeader(outfile5)
+		# self.writeCombinedHeader(outfile6)
+		self.writeCombinedHeader(outfile7)
+		self.writeCombinedHeader(outfile8)
+		
+		self.writeMergedHeader(outfile9)
 		
 		print 'Segmenting using %s method' %options.BOUNDARY
 		for i,ao in enumerate(audio_list):
 			print 'processing: %s' %ao.name
 			

 			tempo_foote_05 = self.pairwiseF(ao.gt, tempo_foote_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
 			tempo_foote_3 = self.pairwiseF(ao.gt, tempo_foote_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
 			timbre_foote_05 = self.pairwiseF(ao.gt, timbre_foote_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
 			timbre_foote_3 = self.pairwiseF(ao.gt, timbre_foote_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
 			
-			self.writeIndividualRes(ao.name, outfile1, gt_novelty_05, gt_novelty_3, harmonic_novelty_05, harmonic_novelty_3, tempo_novelty_05, timbre_novelty_05, timbre_novelty_05, timbre_novelty_3)
-			self.writeIndividualRes(ao.name, outfile2, gt_cnmf_05, gt_cnmf_3, harmonic_cnmf_05, harmonic_cnmf_3, tempo_cnmf_05, timbre_cnmf_05, timbre_cnmf_05, timbre_cnmf_3)
-			self.writeIndividualRes(ao.name, outfile3, gt_sf_05, gt_sf_3, harmonic_sf_05, harmonic_sf_3, tempo_sf_05, timbre_sf_05, timbre_sf_05, timbre_sf_3)
-			self.writeIndividualRes(ao.name, outfile4, gt_foote_05, gt_foote_3, harmonic_foote_05, harmonic_foote_3, tempo_foote_05, timbre_foote_05, timbre_foote_05, timbre_foote_3)
+			self.writeIndividualRes(outfile1, ao.name, gt_novelty_05, gt_novelty_3, harmonic_novelty_05, harmonic_novelty_3, tempo_novelty_05, timbre_novelty_05, timbre_novelty_05, timbre_novelty_3)
+			self.writeIndividualRes(outfile2, ao.name, gt_cnmf_05, gt_cnmf_3, harmonic_cnmf_05, harmonic_cnmf_3, tempo_cnmf_05, timbre_cnmf_05, timbre_cnmf_05, timbre_cnmf_3)
+			self.writeIndividualRes(outfile3, ao.name, gt_sf_05, gt_sf_3, harmonic_sf_05, harmonic_sf_3, tempo_sf_05, timbre_sf_05, timbre_sf_05, timbre_sf_3)
+			self.writeIndividualRes(outfile4, ao.name, gt_foote_05, gt_foote_3, harmonic_foote_05, harmonic_foote_3, tempo_foote_05, timbre_foote_05, timbre_foote_05, timbre_foote_3)
 			
 			
 			############################################################################################################################################
 			# Experiment 2: segmentation using combined features.
 			

 			gt_tb = np.hstack([ao.gammatone_features, ao.timbre_features])
 			gt_tp = np.hstack([ao.gammatone_features, ao.tempo_features])
 			hm_tb = np.hstack([ao.harmonic_features, ao.timbre_features])
 			hm_tp = np.hstack([ao.harmonic_features, ao.tempo_features])
 			tb_tp = np.hstack([ao.timbre_features, ao.tempo_features])
+			gt_hm_tb = np.hstack([ao.gammatone_features, ao.harmonic_features, ao.timbre_features])
 			gt_hm_tp = np.hstack([ao.gammatone_features, ao.harmonic_features, ao.tempo_features])
 			gt_tb_tp = np.hstack([ao.gammatone_features, ao.timbre_features, ao.tempo_features])
 			hm_tb_tp = np.hstack([ao.harmonic_features, ao.timbre_features, ao.tempo_features])
 			gt_hm_tb_tp = np.hstack([ao.gammatone_features, ao.harmonic_features, ao.timbre_features, ao.tempo_features])
 			
+			# Evaluting and writing results.
 			gt_hm_sf_idxs = sf_S.segmentation(gt_hm)
 			gt_tb_sf_idxs = sf_S.segmentation(gt_tb)
 			gt_tp_sf_idxs = sf_S.segmentation(gt_tp)
 			hm_tb_sf_idxs = sf_S.segmentation(hm_tb)
 			hm_tp_sf_idxs = sf_S.segmentation(hm_tp)

 			gt_hm_tp_sf_idxs = sf_S.segmentation(gt_hm_tp)
 			gt_tb_tp_sf_idxs = sf_S.segmentation(gt_tb_tp)
 			hm_tb_tp_sf_idxs = sf_S.segmentation(hm_tb_tp)
 			gt_hm_tb_tp_sf_idxs = sf_S.segmentation(gt_hm_tb_tp)
 			
-			gt_hm_05 = self.pairwiseF(ao.gt, gt_hm_sf_idxs, tolerance=0.5, combine=1.0)
-			gt_tb_05 = self.pairwiseF(ao.gt, gt_tb_sf_idxs, tolerance=0.5, combine=1.0)
-			gt_tp_05 = self.pairwiseF(ao.gt, gt_tp_sf_idxs, tolerance=0.5, combine=1.0)
-			hm_tb_05 = self.pairwiseF(ao.gt, hm_tb_sf_idxs, tolerance=0.5, combine=1.0)
-			hm_tp_05 = self.pairwiseF(ao.gt, hm_tp_sf_idxs, tolerance=0.5, combine=1.0)
-			tb_tp_05 = self.pairwiseF(ao.gt, tb_tp_sf_idxs, tolerance=0.5, combine=1.0)
-			gt_hm_tb_05 = self.pairwiseF(ao.gt, gt_hm_tb_sf_idxs, tolerance=0.5, combine=1.0)
-			gt_hm_tp_05 = self.pairwiseF(ao.gt, gt_hm_tp_sf_idxs, tolerance=0.5, combine=1.0)
-			gt_tb_tp_05 = self.pairwiseF(ao.gt, gt_tb_tp_sf_idxs, tolerance=0.5, combine=1.0)
-			hm_tb_tp_05 = self.pairwiseF(ao.gt, hm_tb_tp_sf_idxs, tolerance=0.5, combine=1.0)
-			gt_hm_tb_tp_05 = self.pairwiseF(ao.gt, gt_hm_tb_tp_sf_idxs, tolerance=0.5, combine=1.0)
-			
-			gt_hm_3 = self.pairwiseF(ao.gt, gt_hm_sf_idxs, tolerance=3, combine=1.0)
-			gt_tb_3 = self.pairwiseF(ao.gt, gt_tb_sf_idxs, tolerance=3, combine=1.0)
-			gt_tp_3 = self.pairwiseF(ao.gt, gt_tp_sf_idxs, tolerance=3, combine=1.0)
-			hm_tb_3 = self.pairwiseF(ao.gt, hm_tb_sf_idxs, tolerance=3, combine=1.0)
-			hm_tp_3 = self.pairwiseF(ao.gt, hm_tp_sf_idxs, tolerance=3, combine=1.0)
-			tb_tp_3 = self.pairwiseF(ao.gt, tb_tp_sf_idxs, tolerance=3, combine=1.0)
-			gt_hm_tb_3 = self.pairwiseF(ao.gt, gt_hm_tb_sf_idxs, tolerance=3, combine=1.0)
-			gt_hm_tp_3 = self.pairwiseF(ao.gt, gt_hm_tp_sf_idxs, tolerance=3, combine=1.0)
-			gt_tb_tp_3 = self.pairwiseF(ao.gt, gt_tb_tp_sf_idxs, tolerance=3, combine=1.0)
-			hm_tb_tp_3 = self.pairwiseF(ao.gt, hm_tb_tp_sf_idxs, tolerance=3, combine=1.0)
-			gt_hm_tb_tp_3 = self.pairwiseF(ao.gt, gt_hm_tb_tp_sf_idxs, tolerance=3, combine=1.0)
+			gt_hm_cnmf_idxs = cnmf_S.segmentation(gt_hm, rank=4, R=R, h=h, niter=300)
+			gt_tb_cnmf_idxs = cnmf_S.segmentation(gt_tb, rank=4, R=R, h=h, niter=300)
+			gt_tp_cnmf_idxs = cnmf_S.segmentation(gt_tp, rank=4, R=R, h=h, niter=300)
+			hm_tb_cnmf_idxs = cnmf_S.segmentation(hm_tb, rank=4, R=R, h=h, niter=300)
+			hm_tp_cnmf_idxs = cnmf_S.segmentation(hm_tp, rank=4, R=R, h=h, niter=300)
+			tb_tp_cnmf_idxs = cnmf_S.segmentation(tb_tp, rank=4, R=R, h=h, niter=300)
+			gt_hm_tb_cnmf_idxs = cnmf_S.segmentation(gt_hm_tb, rank=6, R=R, h=h, niter=300)
+			gt_hm_tp_cnmf_idxs = cnmf_S.segmentation(gt_hm_tp, rank=6, R=R, h=h, niter=300)
+			gt_tb_tp_cnmf_idxs = cnmf_S.segmentation(gt_tb_tp, rank=6, R=R, h=h, niter=300)
+			hm_tb_tp_cnmf_idxs = cnmf_S.segmentation(hm_tb_tp, rank=6, R=R, h=h, niter=300)
+			gt_hm_tb_tp_cnmf_idxs = cnmf_S.segmentation(gt_hm_tb_tp, rank=8, R=R, h=h, niter=300)
+			
+			gt_hm_sf_05 = self.pairwiseF(ao.gt, gt_hm_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_sf_05 = self.pairwiseF(ao.gt, gt_tb_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tp_sf_05 = self.pairwiseF(ao.gt, gt_tp_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_sf_05 = self.pairwiseF(ao.gt, hm_tb_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tp_sf_05 = self.pairwiseF(ao.gt, hm_tp_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			tb_tp_sf_05 = self.pairwiseF(ao.gt, tb_tp_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_sf_05 = self.pairwiseF(ao.gt, gt_hm_tb_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tp_sf_05 = self.pairwiseF(ao.gt, gt_hm_tp_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_tp_sf_05 = self.pairwiseF(ao.gt, gt_tb_tp_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_tp_sf_05 = self.pairwiseF(ao.gt, hm_tb_tp_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_tp_sf_05 = self.pairwiseF(ao.gt, gt_hm_tb_tp_sf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			
+			gt_hm_sf_3 = self.pairwiseF(ao.gt, gt_hm_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_sf_3 = self.pairwiseF(ao.gt, gt_tb_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tp_sf_3 = self.pairwiseF(ao.gt, gt_tp_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_sf_3 = self.pairwiseF(ao.gt, hm_tb_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tp_sf_3 = self.pairwiseF(ao.gt, hm_tp_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			tb_tp_sf_3 = self.pairwiseF(ao.gt, tb_tp_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_sf_3 = self.pairwiseF(ao.gt, gt_hm_tb_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tp_sf_3 = self.pairwiseF(ao.gt, gt_hm_tp_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_tp_sf_3 = self.pairwiseF(ao.gt, gt_tb_tp_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_tp_sf_3 = self.pairwiseF(ao.gt, hm_tb_tp_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_tp_sf_3 = self.pairwiseF(ao.gt, gt_hm_tb_tp_sf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+
+			gt_hm_cnmf_05 = self.pairwiseF(ao.gt, gt_hm_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_cnmf_05 = self.pairwiseF(ao.gt, gt_tb_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tp_cnmf_05 = self.pairwiseF(ao.gt, gt_tp_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_cnmf_05 = self.pairwiseF(ao.gt, hm_tb_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tp_cnmf_05 = self.pairwiseF(ao.gt, hm_tp_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			tb_tp_cnmf_05 = self.pairwiseF(ao.gt, tb_tp_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_cnmf_05 = self.pairwiseF(ao.gt, gt_hm_tb_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tp_cnmf_05 = self.pairwiseF(ao.gt, gt_hm_tp_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_tp_cnmf_05 = self.pairwiseF(ao.gt, gt_tb_tp_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_tp_cnmf_05 = self.pairwiseF(ao.gt, hm_tb_tp_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_tp_cnmf_05 = self.pairwiseF(ao.gt, gt_hm_tb_tp_cnmf_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			
+			gt_hm_cnmf_3 = self.pairwiseF(ao.gt, gt_hm_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_cnmf_3 = self.pairwiseF(ao.gt, gt_tb_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tp_cnmf_3 = self.pairwiseF(ao.gt, gt_tp_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_cnmf_3 = self.pairwiseF(ao.gt, hm_tb_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tp_cnmf_3 = self.pairwiseF(ao.gt, hm_tp_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			tb_tp_cnmf_3 = self.pairwiseF(ao.gt, tb_tp_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_cnmf_3 = self.pairwiseF(ao.gt, gt_hm_tb_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tp_cnmf_3 = self.pairwiseF(ao.gt, gt_hm_tp_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_tb_tp_cnmf_3 = self.pairwiseF(ao.gt, gt_tb_tp_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			hm_tb_tp_cnmf_3 = self.pairwiseF(ao.gt, hm_tb_tp_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			gt_hm_tb_tp_cnmf_3 = self.pairwiseF(ao.gt, gt_hm_tb_tp_cnmf_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
 					
-			self.writeCombinedRes(ao.name, outfile7, gt_hm_05, gt_hm_3, gt_tb_05, gt_tb_3, gt_tp_05, gt_tp_3, hm_tb_05, hm_tb_3, hm_tp_05, hm_tp_3, tb_tp_05, tb_tp_3\
-			gt_hm_tb_05, gt_hm_tb_3, gt_hm_tp_05, gt_hm_tp_3, gt_tb_tp_05, gt_tb_tp_3, hm_tb_tp_05, hm_tb_tp_3, gt_hm_tb_tp_05, gt_hm_tb_tp_3)
+			self.writeCombinedRes(outfile7, ao.name, gt_hm_sf_05, gt_hm_sf_3, gt_tb_sf_05, gt_tb_sf_3, gt_tp_sf_05, gt_tp_sf_3, hm_tb_sf_05, hm_tb_sf_3, hm_tp_sf_05, hm_tp_sf_3, tb_tp_sf_05, tb_tp_sf_3,\
+			gt_hm_tb_sf_05, gt_hm_tb_sf_3, gt_hm_tp_sf_05, gt_hm_tp_sf_3, gt_tb_tp_sf_05, gt_tb_tp_sf_3, hm_tb_tp_sf_05, hm_tb_tp_sf_3, gt_hm_tb_tp_sf_05, gt_hm_tb_tp_sf_3)
+
+			self.writeCombinedRes(outfile8, ao.name, gt_hm_cnmf_05, gt_hm_cnmf_3, gt_tb_cnmf_05, gt_tb_cnmf_3, gt_tp_cnmf_05, gt_tp_cnmf_3, hm_tb_cnmf_05, hm_tb_cnmf_3, hm_tp_cnmf_05, hm_tp_cnmf_3, tb_tp_cnmf_05, tb_tp_cnmf_3,\
+			gt_hm_tb_cnmf_05, gt_hm_tb_cnmf_3, gt_hm_tp_cnmf_05, gt_hm_tp_cnmf_3, gt_tb_tp_cnmf_05, gt_tb_tp_cnmf_3, hm_tb_tp_cnmf_05, hm_tb_tp_cnmf_3, gt_hm_tb_tp_cnmf_05, gt_hm_tb_tp_cnmf_3)
 			
 			############################################################################################################################################
 			# Experiment 3: Pruning boundaries detected by individual boundary algorithms.
 			
-			 
+			# Use different boundary methods for different features
+			gammatone_idxs, harmonic_idxs, timbre_idxs, tempo_idxs = gammatone_sf_idxs, harmonic_sf_idxs, timbre_sf_idxs, tempo_sf_idxs
+			bound_candidates = list(gammatone_idxs) + list(harmonic_idxs) + list(timbre_idxs) + list(tempo_idxs)
+			bound_candidates.sort()
+			
+			nBounds = len(bound_candidates)
+			final_idxs = []
+			idx = 0
+			tol = 10 # tolerance window of merging boundary scores
+			while idx < nBounds:
+				temp = [bound_candidates[idx]]
+				pos = [idx]
+				idx += 1
+				while (idx + tol < nBounds and np.max(bound_candidates[idx: idx+tol]) > 0):
+					temp += [bound_candidates[idx+delta] for delta in xrange(tol) if (bound_candidates[idx]+delta in bound_candidates)]
+					pos += [idx+delta for delta in xrange(tol) if (bound_candidates[idx]+delta in bound_candidates)]
+					idx += tol
+				if len(temp) == 1:
+					final_idxs.append(temp[0])
+				else:
+					final_idxs.append(int(np.rint(np.mean(temp))))
+			
+			merged_05 = self.pairwiseF(ao.gt, final_idxs, tolerance=0.5, combine=1.0, idx2time=ao.ssm_timestamps)
+			merged_3 = self.pairwiseF(ao.gt, final_idxs, tolerance=3, combine=1.0, idx2time=ao.ssm_timestamps)
+			
+			self.writeMergedRes(outfile9, ao.name, merged_05, merged_3)
+			
 			# if options.BOUNDARY == 'novelty':
 			# 	gammatone_novelty, smoothed_gammatone_novelty, gammatone_bound_idxs = novelty_S.process(ao.gammatone_ssm, self.kernel_size, peak_picker)
 			# 	timbre_novelty, smoothed_timbre_novelty, timbre_bound_idxs = novelty_S.process(ao.timbre_ssm, self.kernel_size, peak_picker)
 			# 	tempo_novelty, smoothed_harmonic_novelty, tempo_bound_idxs = novelty_S.process(ao.tempo_ssm, self.kernel_size, peak_picker)
 			# 	harmonic_novelty, smoothed_tempo_novelty, harmonic_bound_idxs = novelty_S.process(ao.harmonic_ssm, self.kernel_size, peak_picker)