Mercurial > hg > audio-time-warp

changeset 14:0ded10fd754c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
part optimised the calculation of combined energy and chrroma in partJointSim calc. Added bug fix
author Andrew N Robertson <andrew.robertson@eecs.qmul.ac.uk>
date Wed, 23 Nov 2011 18:33:17 +0000
parents a3d797046ea8
children 67ed2c7320ba
files src/TimeWarp.cpp src/testApp.cpp src/testApp.h src/timeWarp.cpp
diffstat 4 files changed, 106 insertions(+), 73 deletions(-) [+]
line wrap: on
line diff
--- a/src/TimeWarp.cpp	Tue Nov 22 17:54:48 2011 +0000
+++ b/src/TimeWarp.cpp	Wed Nov 23 18:33:17 2011 +0000
@@ -33,8 +33,8 @@
 //--------------------------------------------------------------
 // destructor
 TimeWarp :: TimeWarp(){
-	diagonalPenalty = 1;//favours diagonal over other paths 
-	diagonalPenalty = 2;//penalises diagonal so all path gradients equal weighting
+	//diagonalPenalty = 1;//favours diagonal over other paths 
+	//diagonalPenalty = 2;//penalises diagonal so all path gradients equal weighting
 	
 }	
 
@@ -69,12 +69,22 @@
 	tmpAlignmentMeasureMatrix.clear();	
 	minimumAlignmentPath.clear();
 	partBackwardsAlignmentPath.clear();
-//	forwardsAlignmentPath.clear();
-	
+	forwardsAlignmentPath.clear();
+	anchorPoints.clear();
 }
 	
 void TimeWarp::initialiseVariables(){
-	diagonalPenalty = 1;
+	similarityMatrix.clear();
+	chromaSimilarityMatrix.clear();
+	tmpSimilarityMatrix.clear();
+	alignmentMeasureMatrix.clear();
+	tmpAlignmentMeasureMatrix.clear();	
+	minimumAlignmentPath.clear();
+	partBackwardsAlignmentPath.clear();
+	forwardsAlignmentPath.clear();
+	anchorPoints.clear();
+	
+	//diagonalPenalty = 1;
 	//chromoGramm.initialise(FRAMESIZE,2048);//framesize 512 and hopsize 2048
 
 }
@@ -444,29 +454,37 @@
 	conversionFactor = (int) round((*firstEnergyVector).size() / (*chromaSimMatrix).size() );
 	simMatrix->clear();
 	
-	double energyProportion = 0.5;
+	double energyProportion = 0.2;
 	double chromaProportion = 1 - energyProportion;
 	double distance, firstSum, secondSum;
 	
 	endX = min (endX, (int)(*firstEnergyVector).size()-1);//in case out of size
-	
+	endY = min( endY+1, (int)(*secondEnergyVector).size());
+	int lastChromaYvalue = 0;
+	int chromaIndexY = 0;
+	double chromaComponent = 0;
+	double 	chromaContribution = 0;
+		DoubleVector d;	
 	for (int x = startX;x <= endX;x++){
-		DoubleVector d;
+		d.clear();
 		
-		for (int y = startY;y < min( endY+1, (int)(*secondEnergyVector).size());y++){
+		//now need the chroma part
+		int chromaIndexX = min(x / conversionFactor, (int)(*chromaSimMatrix).size()-1);
+		
+		for (int y = startY;y < endY;y++){
 			//was thinking to restrict the y part too, but not working yet
+			if (y / conversionFactor != lastChromaYvalue){			
+			chromaIndexY = min(y / conversionFactor, (int)(*chromaSimMatrix)[chromaIndexX].size()-1);
+			chromaComponent = (*chromaSimMatrix)[chromaIndexX][chromaIndexY];
+			lastChromaYvalue = y / conversionFactor;
+			chromaContribution = chromaProportion * chromaComponent;
+			}
 			
 			distance = (*firstEnergyVector)[x] * (*secondEnergyVector)[y];//energy similarity
+			distance *= energyProportion;
+			distance += chromaContribution;
 			
-			//now need the chroma part
-			int chromaIndexX = min(x / conversionFactor, (int)(*chromaSimMatrix).size()-1);
-			int chromaIndexY = min(y / conversionFactor, (int)(*chromaSimMatrix)[chromaIndexX].size()-1);
-			double chromaComponent = (*chromaSimMatrix)[chromaIndexX][chromaIndexY];
-			
-			distance *= energyProportion;
-			distance += chromaProportion * chromaComponent;
-			
-			distance = chromaComponent;
+//			distance = chromaComponent;
 			
 			d.push_back( distance);	
 		}	//end for y
--- a/src/testApp.cpp	Tue Nov 22 17:54:48 2011 +0000
+++ b/src/testApp.cpp	Wed Nov 23 18:33:17 2011 +0000
@@ -133,8 +133,7 @@
 	screenHeight = ofGetHeight() ;
 	screenWidth = ofGetWidth();
 	
-	diagonalPenalty = 2;//penalises diagonal so all path gradients equal weighting
-	
+
 	initialiseVariables();
 	
 	
@@ -144,6 +143,8 @@
 
 void testApp::calculateSimilarityAndAlignment(){
 	
+	tw.initialiseVariables();
+	
 	//here is the main TimeWarp similarity matrix calc, the minimum alignment matrix via dtw and then the backwards path estimate 
 	double timeBefore = ofGetElapsedTimef();
 	tw.calculateChromaSimilarityMatrix(&tw.chromaMatrix,  &tw.secondMatrix, &tw.chromaSimilarityMatrix);
@@ -154,9 +155,13 @@
 
 	calculateForwardsAlignment();
 	
+	doPathBugCheck();
+		
 	tw.copyForwardsPathToBackwardsPath();
 	backwardsAlignmentIndex = tw.backwardsAlignmentPath[0].size()-1;
 	
+
+	
 //		tw.calculateAlignmentMatrix(tw.firstChromaEnergyMatrix, tw.secondChromaEnergyMatrix, &tw.alignmentMeasureMatrix);								
 //		tw.calculateMinimumAlignmentPath(&tw.alignmentMeasureMatrix, &tw.backwardsAlignmentPath, false);
 //	tw.calculatePartMinimumAlignmentPath(0, 0, 100, 100, &tw.partAlignmentMeasureMatrix);
@@ -166,6 +171,14 @@
 	printVariousMatrixInfo();
 }
 
+void testApp::doPathBugCheck(){
+//bug check
+if (tw.forwardsAlignmentPath.size() > 0 && tw.forwardsAlignmentPath[0][1] != 0){
+	tw.forwardsAlignmentPath[0][1] = 0;//sometimes is large rndm number
+	printf("BUG IN FORWARDS PATH FIXED!!!\n");
+}
+}
+
 
 
 void testApp::setConversionRatio(){
@@ -190,8 +203,8 @@
 	printf("backwards path size is [0]:%i, [1]%i, \n", (int)tw.backwardsAlignmentPath[0].size(), (int)tw.backwardsAlignmentPath[1].size());
 	
 	printf("backwards path size is %i, FORWARDS SIZE IS %i\n", (int)tw.backwardsAlignmentPath[0].size(), (int)tw.forwardsAlignmentPath[0].size());
-	printf("BACKWARDS PATH::\n");
-	tw.printBackwardsPath(0, (int)tw.backwardsAlignmentPath[0].size(), &tw.backwardsAlignmentPath);
+	//printf("BACKWARDS PATH::\n");
+	//tw.printBackwardsPath(0, (int)tw.backwardsAlignmentPath[0].size(), &tw.backwardsAlignmentPath);
 }
 
 void testApp::initialiseVariables(){
@@ -209,7 +222,7 @@
 	audioPlaying = false;
 	audioPaused = true;
 	
-	tw.initialiseVariables();
+//	tw.initialiseVariables();
 	
 }
 
@@ -218,8 +231,8 @@
 	
 	tw.forwardsAlignmentPath.clear();
 	//causal part
-	int hopsize = 200;
-	int frameSize = 600;
+	int hopsize = 400;
+	int frameSize = 800;
 	int startFrameY = 0;
 	int startFrameX = 0;
 	tw.anchorPoints.clear();
@@ -239,9 +252,9 @@
 		
 		
 		
-		printf("TMP size of tmp sim is %i\n", (int)tw.tmpSimilarityMatrix.size());	
+	//	printf("TMP size of tmp sim is %i\n", (int)tw.tmpSimilarityMatrix.size());	
 		double elapsedTime = ofGetElapsedTimef() - timeBefore;
-		printf("PART SIM MATRIX CAL TAKES %f\n", elapsedTime);
+	//	printf("PART SIM MATRIX CAL TAKES %f\n", elapsedTime);
 		
 		//change part sim to have a y limit too
 		
@@ -1088,7 +1101,7 @@
 	int readcount = 1; // counts number of samples read from sound file
 	printf("processing audio from doublematrix \n");
 			printf("readcount %i", readcount);
-	while(readcount != 0 && moveOn == true)
+	while(readcount != 0 && moveOn == true && energyVector->size() < FILE_LIMIT)
 	{
 		
 		// read FRAMESIZE samples from 'infile' and save in 'data'
@@ -1252,10 +1265,14 @@
 
 	}
 	
-	if (key == 'r'){
+	if (key == 'f'){
 		tw.printBackwardsPath(0, (int) tw.forwardsAlignmentPath[0].size(), &tw.forwardsAlignmentPath);
 	}
 	
+	if (key == 'b'){
+		tw.printBackwardsPath(0, (int) tw.backwardsAlignmentPath[0].size(), &tw.backwardsAlignmentPath);
+	}
+	
 	
 	if (key == 's'){
 		drawSimilarity = !drawSimilarity;
@@ -1414,22 +1431,9 @@
 
 
 void testApp::loadFirstAudioFile(){
-/*
- Chromagram* cptr;
-	DoubleMatrix* dptr;
-	DoubleVector* eptr;
-	
-	cptr = &(tw.chromoGramm);
-	dptr = &(tw.chromaMatrix);
-	eptr = &(tw.firstEnergyVector);
-	*/
+
 	processAudioToDoubleMatrix(&tw.chromaMatrix, &tw.firstEnergyVector);
-	/*
-	double timeBefore = ofGetElapsedTimef();
-	tw.createCombinedMatrix(tw.chromaMatrix, tw.firstEnergyVector, &tw.firstChromaEnergyMatrix);
-	double elapsedTime = ofGetElapsedTimef() - timeBefore;
-	printf("CREATING JOINT MATRIX for FIRST ENERGY and chroma TAKES %f\n", elapsedTime);
-	*/
+
 	
 }
 
@@ -1440,16 +1444,8 @@
 	const char	*infilenme = sndFileName.c_str() ;	
 	loadLibSndFile(infilenme);
 
-/*	Chromagram* cptr;
-	DoubleMatrix* dptr;
-	DoubleVector* eptr;
-	cptr = &(tw.secondChromoGramm);
-	dptr = &(tw.secondMatrix);
-	eptr = &(tw.secondEnergyVector);
- */
-	
-	processAudioToDoubleMatrix(&tw.secondMatrix, &tw.secondEnergyVector);//dptr, e ptr);//chromaMatrix);
-//	tw.createCombinedMatrix(tw.secondMatrix, tw.secondEnergyVector, &tw.secondChromaEnergyMatrix);
+
+	processAudioToDoubleMatrix(&tw.secondMatrix, &tw.secondEnergyVector);
 	
 }
 
--- a/src/testApp.h	Tue Nov 22 17:54:48 2011 +0000
+++ b/src/testApp.h	Wed Nov 23 18:33:17 2011 +0000
@@ -19,7 +19,7 @@
 #define CHROMA_LENGTH 12000
 #define CHROMA_CONVERSION_FACTOR 16 //16 times as many frames in energy as in chroma
 #define CHROMAGRAM_FRAMESIZE 2048
-
+#define FILE_LIMIT 10000
 
 //length in terms of frames (at 512 samples per frame - there are 90 per second) => 900: 10 seconds
 
@@ -78,6 +78,7 @@
 	void printSimilarityMatrix(int sizeToPrint);
 	void setConversionRatio();
 	void printVariousMatrixInfo();
+	void doPathBugCheck();
 	
 	void drawChromaSimilarityMatrix();
 	
--- a/src/timeWarp.cpp	Tue Nov 22 17:54:48 2011 +0000
+++ b/src/timeWarp.cpp	Wed Nov 23 18:33:17 2011 +0000
@@ -33,8 +33,8 @@
 //--------------------------------------------------------------
 // destructor
 TimeWarp :: TimeWarp(){
-	diagonalPenalty = 1;//favours diagonal over other paths 
-	diagonalPenalty = 2;//penalises diagonal so all path gradients equal weighting
+	//diagonalPenalty = 1;//favours diagonal over other paths 
+	//diagonalPenalty = 2;//penalises diagonal so all path gradients equal weighting
 	
 }	
 
@@ -69,12 +69,22 @@
 	tmpAlignmentMeasureMatrix.clear();	
 	minimumAlignmentPath.clear();
 	partBackwardsAlignmentPath.clear();
-//	forwardsAlignmentPath.clear();
-	
+	forwardsAlignmentPath.clear();
+	anchorPoints.clear();
 }
 	
 void TimeWarp::initialiseVariables(){
-	diagonalPenalty = 1;
+	similarityMatrix.clear();
+	chromaSimilarityMatrix.clear();
+	tmpSimilarityMatrix.clear();
+	alignmentMeasureMatrix.clear();
+	tmpAlignmentMeasureMatrix.clear();	
+	minimumAlignmentPath.clear();
+	partBackwardsAlignmentPath.clear();
+	forwardsAlignmentPath.clear();
+	anchorPoints.clear();
+	
+	//diagonalPenalty = 1;
 	//chromoGramm.initialise(FRAMESIZE,2048);//framesize 512 and hopsize 2048
 
 }
@@ -444,29 +454,37 @@
 	conversionFactor = (int) round((*firstEnergyVector).size() / (*chromaSimMatrix).size() );
 	simMatrix->clear();
 	
-	double energyProportion = 0.5;
+	double energyProportion = 0.2;
 	double chromaProportion = 1 - energyProportion;
 	double distance, firstSum, secondSum;
 	
 	endX = min (endX, (int)(*firstEnergyVector).size()-1);//in case out of size
-	
+	endY = min( endY+1, (int)(*secondEnergyVector).size());
+	int lastChromaYvalue = 0;
+	int chromaIndexY = 0;
+	double chromaComponent = 0;
+	double 	chromaContribution = 0;
+		DoubleVector d;	
 	for (int x = startX;x <= endX;x++){
-		DoubleVector d;
+		d.clear();
 		
-		for (int y = startY;y < min( endY+1, (int)(*secondEnergyVector).size());y++){
+		//now need the chroma part
+		int chromaIndexX = min(x / conversionFactor, (int)(*chromaSimMatrix).size()-1);
+		
+		for (int y = startY;y < endY;y++){
 			//was thinking to restrict the y part too, but not working yet
+			if (y / conversionFactor != lastChromaYvalue){			
+			chromaIndexY = min(y / conversionFactor, (int)(*chromaSimMatrix)[chromaIndexX].size()-1);
+			chromaComponent = (*chromaSimMatrix)[chromaIndexX][chromaIndexY];
+			lastChromaYvalue = y / conversionFactor;
+			chromaContribution = chromaProportion * chromaComponent;
+			}
 			
 			distance = (*firstEnergyVector)[x] * (*secondEnergyVector)[y];//energy similarity
+			distance *= energyProportion;
+			distance += chromaContribution;
 			
-			//now need the chroma part
-			int chromaIndexX = min(x / conversionFactor, (int)(*chromaSimMatrix).size()-1);
-			int chromaIndexY = min(y / conversionFactor, (int)(*chromaSimMatrix)[chromaIndexX].size()-1);
-			double chromaComponent = (*chromaSimMatrix)[chromaIndexX][chromaIndexY];
-			
-			distance *= energyProportion;
-			distance += chromaProportion * chromaComponent;
-			
-			distance = chromaComponent;
+//			distance = chromaComponent;
 			
 			d.push_back( distance);	
 		}	//end for y