annotate dsp/segmentation/cluster_segmenter.c @ 321:f1e6be2de9a5

A threshold (delta) is added in the peak picking parameters structure (PPickParams). It is used as an offset when computing the smoothed detection function. A constructor for the structure PPickParams is also added to set the parameters to 0 when a structure instance is created. Hence programmes using the peak picking parameter structure and which do not set the delta parameter (e.g. QM Vamp note onset detector) won't be affected by the modifications.

Functions modified:
- dsp/onsets/PeakPicking.cpp
- dsp/onsets/PeakPicking.h
- dsp/signalconditioning/DFProcess.cpp
- dsp/signalconditioning/DFProcess.h

/*
 *  cluster_segmenter.c
 *  soundbite
 *
 *  Created by Mark Levy on 06/04/2006.
 *  Copyright 2006 Centre for Digital Music, Queen Mary, University of London.

    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License as
    published by the Free Software Foundation; either version 2 of the
    License, or (at your option) any later version.  See the file
    COPYING included with this distribution for more information.
 *
 */

#include "cluster_segmenter.h"

extern int readmatarray_size(const char *filepath, int n_array, int* t, int* d);
extern int readmatarray(const char *filepath, int n_array, int t, int d, double** arr);

/* converts constant-Q features to normalised chroma */
void cq2chroma(double** cq, int nframes, int ncoeff, int bins, double** chroma)
{
	int noct = ncoeff / bins;	/* number of complete octaves in constant-Q */
	int t, b, oct, ix;
	//double maxchroma;	/* max chroma value at each time, for normalisation */
	//double sum;		/* for normalisation */
	
	for (t = 0; t < nframes; t++)
	{
		for (b = 0; b < bins; b++)
			chroma[t][b] = 0;
		for (oct = 0; oct < noct; oct++)
		{
			ix = oct * bins;
			for (b = 0; b < bins; b++)
				chroma[t][b] += fabs(cq[t][ix+b]);
		}
		/* normalise to unit sum
		sum = 0;
		for (b = 0; b < bins; b++)
			sum += chroma[t][b];
		for (b = 0; b < bins; b++)
			chroma[t][b] /= sum;
		*/
		/* normalise to unit max - NO this made results much worse!
		maxchroma = 0;
		for (b = 0; b < bins; b++)
			if (chroma[t][b] > maxchroma)
				maxchroma = chroma[t][b];
		if (maxchroma > 0)
			for (b = 0; b < bins; b++)
				chroma[t][b] /= maxchroma;	
		*/
	}
}

/* applies MPEG-7 normalisation to constant-Q features, storing normalised envelope (norm) in last feature dimension */
void mpeg7_constq(double** features, int nframes, int ncoeff)
{
	int i, j;
	double ss;
	double env;
	double maxenv = 0;
	
	/* convert const-Q features to dB scale */
	for (i = 0; i < nframes; i++)
		for (j = 0; j < ncoeff; j++)
			features[i][j] = 10.0 * log10(features[i][j]+DBL_EPSILON);
	
	/* normalise each feature vector and add the norm as an extra feature dimension */	
	for (i = 0; i < nframes; i++)
	{
		ss = 0;
		for (j = 0; j < ncoeff; j++)
			ss += features[i][j] * features[i][j];
		env = sqrt(ss);
		for (j = 0; j < ncoeff; j++)
			features[i][j] /= env;
		features[i][ncoeff] = env;
		if (env > maxenv)
			maxenv = env;
	} 
	/* normalise the envelopes */
	for (i = 0; i < nframes; i++)
		features[i][ncoeff] /= maxenv;	
}

/* return histograms h[nx*m] of data x[nx] into m bins using a sliding window of length h_len (MUST BE ODD) */
/* NB h is a vector in row major order, as required by cluster_melt() */
/* for historical reasons we normalise the histograms by their norm (not to sum to one) */
void create_histograms(int* x, int nx, int m, int hlen, double* h)
{
	int i, j, t;
	double norm;

	for (i = 0; i < nx*m; i++) 
	        h[i] = 0;

	for (i = hlen/2; i < nx-hlen/2; i++)
	{
		for (j = 0; j < m; j++)
			h[i*m+j] = 0;
		for (t = i-hlen/2; t <= i+hlen/2; t++)
			++h[i*m+x[t]];
		norm = 0;
		for (j = 0; j < m; j++)
			norm += h[i*m+j] * h[i*m+j];
		for (j = 0; j < m; j++)
			h[i*m+j] /= norm;
	}
	
	/* duplicate histograms at beginning and end to create one histogram for each data value supplied */
	for (i = 0; i < hlen/2; i++)
		for (j = 0; j < m; j++)
			h[i*m+j] = h[hlen/2*m+j];
	for (i = nx-hlen/2; i < nx; i++)
		for (j = 0; j < m; j++)
			h[i*m+j] = h[(nx-hlen/2-1)*m+j];
}

/* segment using HMM and then histogram clustering */
void cluster_segment(int* q, double** features, int frames_read, int feature_length, int nHMM_states, 
					 int histogram_length, int nclusters, int neighbour_limit)
{
	int i, j;
	
	/*****************************/
	if (0) {
	/* try just using the predominant bin number as a 'decoded state' */
	nHMM_states = feature_length + 1;	/* allow a 'zero' state */
	double chroma_thresh = 0.05;
	double maxval;
	int maxbin;
	for (i = 0; i < frames_read; i++)
	{
		maxval = 0;
		for (j = 0; j < feature_length; j++)
		{
			if (features[i][j] > maxval) 
			{
				maxval = features[i][j];
				maxbin = j;
			}				
		}
		if (maxval > chroma_thresh)
			q[i] = maxbin;
		else
			q[i] = feature_length;
	}
	
	}
	if (1) {
	/*****************************/
		
	
	/* scale all the features to 'balance covariances' during HMM training */
	double scale = 10;
	for (i = 0; i < frames_read; i++)
		for (j = 0; j < feature_length; j++)
			features[i][j] *= scale;
	
	/* train an HMM on the features */
	
	/* create a model */
	model_t* model = hmm_init(features, frames_read, feature_length, nHMM_states);
	
	/* train the model */
	hmm_train(features, frames_read, model);
/*	
	printf("\n\nafter training:\n");
	hmm_print(model);
*/	
	/* decode the hidden state sequence */
	viterbi_decode(features, frames_read, model, q);  
	hmm_close(model);
	
	/*****************************/
	}
	/*****************************/
	
    
/*
	fprintf(stderr, "HMM state sequence:\n");
	for (i = 0; i < frames_read; i++)
		fprintf(stderr, "%d ", q[i]);
	fprintf(stderr, "\n\n");
*/
	
	/* create histograms of states */
	double* h = (double*) malloc(frames_read*nHMM_states*sizeof(double));	/* vector in row major order */
	create_histograms(q, frames_read, nHMM_states, histogram_length, h);
	
	/* cluster the histograms */
	int nbsched = 20;	/* length of inverse temperature schedule */
	double* bsched = (double*) malloc(nbsched*sizeof(double));	/* inverse temperature schedule */
	double b0 = 100;
	double alpha = 0.7;
	bsched[0] = b0;
	for (i = 1; i < nbsched; i++)
		bsched[i] = alpha * bsched[i-1];
	cluster_melt(h, nHMM_states, frames_read, bsched, nbsched, nclusters, neighbour_limit, q);
	
	/* now q holds a sequence of cluster assignments */
	
	free(h);  
	free(bsched);
}

/* segment constant-Q or chroma features */
void constq_segment(int* q, double** features, int frames_read, int bins, int ncoeff, int feature_type, 
			 int nHMM_states, int histogram_length, int nclusters, int neighbour_limit)
{
	int feature_length;
	double** chroma;
	int i;
	
	if (feature_type == FEATURE_TYPE_CONSTQ)
	{
/*		fprintf(stderr, "Converting to dB and normalising...\n");
 */		
		mpeg7_constq(features, frames_read, ncoeff);
/*		
		fprintf(stderr, "Running PCA...\n");
*/		
		/* do PCA on the features (but not the envelope) */
		int ncomponents = 20;
		pca_project(features, frames_read, ncoeff, ncomponents);
		
		/* copy the envelope so that it immediatly follows the chosen components */
		for (i = 0; i < frames_read; i++)
			features[i][ncomponents] = features[i][ncoeff];	
		
		feature_length = ncomponents + 1;
		
		/**************************************
		//TEST
		// feature file name
		char* dir = "/Users/mark/documents/semma/audio/";
		char* file_name = (char*) malloc((strlen(dir) + strlen(trackname) + strlen("_features_c20r8h0.2f0.6.mat") + 1)*sizeof(char));
		strcpy(file_name, dir);
		strcat(file_name, trackname);
		strcat(file_name, "_features_c20r8h0.2f0.6.mat");
		
		// get the features from Matlab from mat-file
		int frames_in_file;
		readmatarray_size(file_name, 2, &frames_in_file, &feature_length);
		readmatarray(file_name, 2, frames_in_file, feature_length, features);
		// copy final frame to ensure that we get as many as we expected
		int missing_frames = frames_read - frames_in_file;
		while (missing_frames > 0)
		{
			for (i = 0; i < feature_length; i++)
				features[frames_read-missing_frames][i] = features[frames_read-missing_frames-1][i];
			--missing_frames;
		}
		
		free(file_name);
		******************************************/
	
		cluster_segment(q, features, frames_read, feature_length, nHMM_states, histogram_length, nclusters, neighbour_limit);
	}
	
	if (feature_type == FEATURE_TYPE_CHROMA)
	{
/*
		fprintf(stderr, "Converting to chroma features...\n");
*/		
		/* convert constant-Q to normalised chroma features */
		chroma = (double**) malloc(frames_read*sizeof(double*));
		for (i = 0; i < frames_read; i++)
			chroma[i] = (double*) malloc(bins*sizeof(double));
		cq2chroma(features, frames_read, ncoeff, bins, chroma);
		feature_length = bins;
		
		cluster_segment(q, chroma, frames_read, feature_length, nHMM_states, histogram_length, nclusters, neighbour_limit);
	
		for (i = 0; i < frames_read; i++)
			free(chroma[i]);
		free(chroma);
	}
}