Mercurial > hg > multitrack-audio-matcher

changeset 10:cbadb9d05d29

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Using timestamps for the scrolling alignment time
author Andrew N Robertson <andrew.robertson@eecs.qmul.ac.uk>
date Sat, 04 Feb 2012 19:59:27 +0000
parents bc62266af280
children 9a2b008c4706
files README.txt bayesianArraySrc/BayesianArrayStructure.cpp src/AudioEventMatcher.cpp src/AudioEventMatcher.h src/RecordedMultitrackAudio.cpp src/testApp.cpp
diffstat 6 files changed, 228 insertions(+), 34 deletions(-) [+]
line wrap: on
line diff
--- a/README.txt	Fri Feb 03 17:53:14 2012 +0000
+++ b/README.txt	Sat Feb 04 19:59:27 2012 +0000
@@ -24,3 +24,15 @@
 Looking at offset correction - appears offset changes after bestUpdate but our MAP and dustbin might be for the last offset
 
 changing crossupdate threshold UP for tmp bug
+
+Look at audio pitch - is this reliable? - do we have other alternatives?
+
+Harmonic scale could play a part - pitch -> note
+
+Accuracy of the onset detector. Can we get more precise about where the onset actually is, both in analysis and the live frame? (PAPER?)
+
+OPTIMISE the starting point for the update routine when new`Pitch or onset is found. goes all the way through chromaOnsets
+
+
+
+
--- a/bayesianArraySrc/BayesianArrayStructure.cpp	Fri Feb 03 17:53:14 2012 +0000
+++ b/bayesianArraySrc/BayesianArrayStructure.cpp	Sat Feb 04 19:59:27 2012 +0000
@@ -211,7 +211,7 @@
 void BayesianArrayStructure::updateBestEstimate(const double& timeDifference){
 //	double timeDiff = ofGetElapsedTimeMillis() - lastEventTime;//
 	double tmp = bestEstimate;
-	printf("best est routine: posterior offset %f\n", posterior.offset);
+//	printf("best est routine: posterior offset %f\n", posterior.offset);
 	
 	double timeDiff = timeDifference;
 	
@@ -232,10 +232,12 @@
 	speedEstimate = relativeSpeedPosterior.getIndexInRealTerms(speedEstimateIndex);
 	bestEstimate = posterior.getIndexInRealTerms(posterior.MAPestimate) + timeDiff*speedEstimate;
 	
-	printf("best estimate update from %f to %f; time diff %f MAP %i = %f ms speed index %f est %f SpeedxTime %f\n", tmp, bestEstimate, timeDiff,
-		   posterior.MAPestimate, posterior.getIndexInRealTerms(posterior.MAPestimate), speedEstimateIndex, speedEstimate, timeDiff*speedEstimate);
+//	printf("best estimate update from %f to %f; time diff %f MAP %i = %f ms speed index %f est %f SpeedxTime %f\n", tmp, bestEstimate, timeDiff,
+//		   posterior.MAPestimate, posterior.getIndexInRealTerms(posterior.MAPestimate), speedEstimateIndex, speedEstimate, timeDiff*speedEstimate);
+	printf("BEST ESTIMATE %f\n", bestEstimate);
 }
 
+
 void BayesianArrayStructure::calculatePosterior(){
 	//posterior.doProduct(prior, likelihood);
 	assert(posterior.offset == prior.offset);
@@ -284,7 +286,7 @@
 //	int tmpMap = bayesianStruct.posterior.getMAPestimate();
 	
 	double timeDifference = newEventTime - lastEventTime; 
-//	printf("updating distributions at time %f diff %f offset %f tmpmap est %i\n", newEventTime, timeDifference, bayesianStruct.posterior.offset, tmpMap);
+	printf("updating distributions at time %f diff %f offset %f\n", newEventTime, timeDifference, posterior.offset);
 	
 	//addnoise to the tempo distribution
 	//bayesianStruct.decaySpeedDistribution(timeDifference);
@@ -314,6 +316,8 @@
 }
 
 
+
+
 void BayesianArrayStructure::crossUpdateArrays(DynamicVector& position, DynamicVector& speed, double timeDifference){
 
 	//set the cutoff for offset of position first! XXX
@@ -657,6 +661,155 @@
 	printf(" MAP index %i post offset %f == %f ms\n", posterior.MAPestimate, posterior.offset, posterior.getIndexInRealTerms(posterior.MAPestimate));
 }
 
+//PROJECT PRIOR CODE
+/*
+void BayesianArrayStructure::projectDistribution(const double& newEventTime, DynamicVector& projectedPrior){
+	
+	projectedPrior.copyFromDynamicVector(posterior);
+
+	double timeDifference = newEventTime - lastEventTime; 
+	
+//	if (timeDifference > 50 && updatingSpeedDistribution){
+//		addGaussianNoiseToSpeedPosterior(timeDifference * 10.0 / 100.);
+//	}
+
+	
+	updateBestEstimate(timeDifference);
+	lastBestEstimateUpdateTime = newEventTime;//getTimeNow(timePlayed);
+	
+	//set TARGETS - commented tenmporarily
+	setNewDistributionOffsets(max(0., bestEstimate - (prior.scalar*prior.arraySize/2)));
+	
+	crossUpdateArrays(posterior, relativeSpeedPosterior, timeDifference);
+	
+	//i.e. using the same offset as prior
+	posterior.offset = prior.offset;// 
+	
+	//	float tmpPrior = max(0., bestEstimate - (prior.scalar*prior.arraySize/2));//	prior.offset = max(0., bestEstimate - (prior.scalar*prior.arraySize/2));
+	//	printf("Using prior offset of %f not %f\n", tmpPrior, prior.offset);
+	
+	lastEventTime = newEventTime;//lastEventTime = ofGetElapsedTimeMillis();
+	
+}
+
+
+
+
+void BayesianArrayStructure::crossUpdateArrays(DynamicVector& position, DynamicVector& speed, double timeDifference){
+	
+	//set the cutoff for offset of position first! XXX
+	
+	double timeDifferenceInPositionVectorUnits = timeDifference / prior.scalar;
+	
+	printf("CROSS UPDATE time diff %f ms is %f units; ", timeDifference, timeDifferenceInPositionVectorUnits);
+	prior.zero();//kill prior
+	
+	//	calculateNewPriorOffset(timeDifference);//dioesnt do anything
+	
+	//	printf("new prior offset %f and post offset %f\n", prior.offset, posterior.offset);
+	
+	if (timeDifferenceInPositionVectorUnits > crossUpdateTimeThreshold)
+		complexCrossUpdate(timeDifferenceInPositionVectorUnits);
+	else
+		translateByMaximumSpeed(timeDifferenceInPositionVectorUnits);	
+	
+	
+	updateCounter++;
+	prior.renormalise();//not strictly necessary??
+	
+}
+
+void BayesianArrayStructure::complexCrossUpdate(const double& timeDifferenceInPositionVectorUnits){
+	
+	
+	printf("before cross c : posterior map is %i = %f ms time diff pos vec %f\n", posterior.getMAPestimate(), posterior.getIndexInRealTerms(prior.getMAPestimate()), timeDifferenceInPositionVectorUnits);
+	
+	int distanceMoved, newPriorIndex;
+	
+	double speedValue = relativeSpeedPosterior.offset;
+	
+	for (int i = 0;i < relativeSpeedPosterior.arraySize;i++){
+		
+		//	double speedValue = relativeSpeedPosterior.getIndexInRealTerms(i);//so for scalar 0.01, 50 -> speed value of 0.5
+		
+		//so we have moved 
+		distanceMoved = round(timeDifferenceInPositionVectorUnits * speedValue);//round the value
+		//	printf("Speed value %f time %f gives distance %i\n", speedValue, timeDifferenceInPositionVectorUnits, distanceMoved);
+		
+		if (relativeSpeedPosterior.array[i] != 0){
+			
+			double speedContribution = relativeSpeedPosterior.array[i];
+			
+			//	printf("speed [%i](val[%f]) gives %f moved %i in %f units \n", i, relativeSpeedPosterior.array[i], speedValue, distanceMoved, timeDifferenceInPositionVectorUnits);
+			
+			//1/2/12 deleted line
+			newPriorIndex = posterior.offset - prior.offset + distanceMoved;//i.e. where post[0] goes to in terms of prior at this speed
+			int postIndex = 0;//index of posterior that will contribute
+			
+			while (postIndex < posterior.arraySize && newPriorIndex < prior.arraySize){
+				
+				//did use a for loop
+				//	for (postIndex = 0;postIndex < posterior.arraySize;postIndex++){
+				//old posterior contributing to new prior
+				
+				//would use this method
+				//newPriorIndex = postIndex + posterior.offset - prior.offset + distanceMoved;
+				
+				if (newPriorIndex >= 0){
+					prior.addToIndex(newPriorIndex, posterior.array[postIndex]*speedContribution);
+					//		printf("speed index %i new prior index %i post val %f speed contrib %f dist %i\n", i, newPriorIndex, posterior.array[postIndex], speedContribution, distanceMoved);
+				}
+				//but we actually do this for simplicity
+				newPriorIndex++;
+				postIndex++;
+			}//end for. now while
+			
+			
+		}//if not zero
+		
+		speedValue += relativeSpeedPosterior.scalar;//optimised line
+		//as we wanted:
+		//	double speedValue = relativeSpeedPosterior.getIndexInRealTerms(i);//so for scalar 0.01, 50 -> speed value of 0.5
+		
+	}//end speed
+	
+	
+	printf("after cross c : prior map is %i = %f ms\n", prior.getMAPestimate(), prior.getIndexInRealTerms(prior.getMAPestimate()));
+}
+
+
+
+void BayesianArrayStructure::translateByMaximumSpeed(const double& timeDifferenceInPositionVectorUnits){
+	
+	
+	int distanceMoved, newPriorIndex;
+	double speedIndex = getSpeedEstimateIndex();
+	double speedValue = relativeSpeedPosterior.getIndexInRealTerms(speedIndex);
+	
+	//	double speedValue = relativeSpeedPosterior.getIndexInRealTerms(relativeSpeedPosterior.integratedEstimate);
+	
+	//so for scalar 0.01, 50 -> speed value of 0.5
+	double speedContribution = relativeSpeedPosterior.array[(int)round(speedIndex)];
+	//so we have moved 
+	distanceMoved = round(timeDifferenceInPositionVectorUnits * speedValue);//round the value
+	
+	//	printf("speed [%i] gives %f moved %i in %f units \n", i, speedValue, distanceMoved, timeDifferenceInPositionVectorUnits);
+	
+	for (int postIndex = 0;postIndex < posterior.arraySize;postIndex++){
+		//old posterior contributing to new prior
+		newPriorIndex = postIndex + posterior.offset - prior.offset + distanceMoved;
+		if (newPriorIndex >= 0 && newPriorIndex < prior.arraySize){
+			prior.addToIndex(newPriorIndex, posterior.array[postIndex]*speedContribution);
+		}
+		
+	}
+	
+}
+ */
+
+//END PROJECT PRIOR CODE
+
+
 /*
  
  void BayesianArrayStructure::updateTempoDistribution(const double& speedRatio, const double& matchFactor){
--- a/src/AudioEventMatcher.cpp	Fri Feb 03 17:53:14 2012 +0000
+++ b/src/AudioEventMatcher.cpp	Sat Feb 04 19:59:27 2012 +0000
@@ -65,8 +65,18 @@
 }
 
 void AudioEventMatcher::updateBestAlignmentPosition(){
+	//THIS DEALS WITH WHERE WE ARE NOW! ON THE SCREEN
+	//DIFFERENT TO WHEN EVENTS COME IN AS THEY ARE TIMESTAMPED - SO EG A PITCH EVENT MAY ARRIVE 16 CHROMA FRAMES LATER - BIG DIFFERENCE
+	
+	int newTime = ofGetElapsedTimeMillis() - startTime;
+//	double tmp = bayesianStruct.posterior.getIndexInRealTerms(bayesianStruct.posterior.MAPestimate);;
+//	double timetmp = (newTime - lastAlignmentTime);
+//	double speedtmp = bayesianStruct.relativeSpeedPosterior.getIndexInRealTerms(bayesianStruct.relativeSpeedPosterior.MAPestimate);
+	
 	currentAlignmentPosition = bayesianStruct.posterior.getIndexInRealTerms(bayesianStruct.posterior.MAPestimate);
-	currentAlignmentPosition += (ofGetElapsedTimeMillis() - lastAlignmentTime) * bayesianStruct.relativeSpeedPosterior.getIndexInRealTerms(bayesianStruct.relativeSpeedPosterior.MAPestimate);
+	currentAlignmentPosition += (newTime - lastAlignmentTime) * bayesianStruct.relativeSpeedPosterior.getIndexInRealTerms(bayesianStruct.relativeSpeedPosterior.MAPestimate);
+
+//	printf("ALIGN pos %f time diff %f (now %f , last %f)speed %f :: ALIGN BEST %f\n", tmp, timetmp, (double)ofGetElapsedTimeMillis(), lastAlignmentTime, speedtmp, currentAlignmentPosition);
 }
 
 void AudioEventMatcher::draw(){
@@ -153,11 +163,10 @@
 
 	int priorStartIndex = recentPrior.getRealTermsAsIndex(screenStartTimeMillis);
 	int priorEndIndex = recentPrior.getRealTermsAsIndex(screenEndTimeMillis);
-	ofSetColor(0,200,200);
+	ofSetColor(0,200,200);//recent prior
 	recentPrior.drawConstrainedVector(priorStartIndex, priorEndIndex, 0, ofGetWidth(), bayesPositionWindow);
 
-//	bayesianStruct.prior.addTriangularShape(100, 20, 0.4);
-	ofSetColor(255,0,100);
+	ofSetColor(255,0,100);//purple prior
 	bayesianStruct.prior.drawConstrainedVector(bayesianStruct.prior.getRealTermsAsIndex(screenStartTimeMillis), bayesianStruct.prior.getRealTermsAsIndex(screenEndTimeMillis), 0, ofGetWidth(), bayesLikelihoodWindow);
 	
 }
@@ -166,11 +175,8 @@
 	if (pitchIn > 0){
 	liveInput.addPitchEvent(pitchIn, timeIn);
 	
-	//tmp print stuff
-	printf("New pitch MAP post estimate now %i, ", bayesianStruct.posterior.MAPestimate);
-	double tmp  = bayesianStruct.posterior.getMAPestimate();
-	printf(" getting it %f and offset %f == %f ms\n", tmp,  bayesianStruct.posterior.offset, bayesianStruct.posterior.getIndexInRealTerms(tmp));
-	
+	//printPosteriorMAPinfo();	
+		
 	matchNewPitchEvent(channel, pitchIn, timeIn);//main pitch matching fn
 
 	likelihoodVisualisation[1] = bayesianStruct.likelihood;
@@ -179,7 +185,6 @@
 	recentTime = timeIn;
 	}
 	
-	
 }
 
 void AudioEventMatcher::newKickEvent(const double& timeIn){	
@@ -209,9 +214,8 @@
 //Needs just to set bounds for the matching process, not have TimeIn
 void AudioEventMatcher::matchNewOnsetEvent(const int& channel, const double& timeIn){
 
-	
 	bayesianStruct.updateBayesianDistributions(timeIn);//moves the posterior up into prior given the time interval and calculates new offsets
-	
+
 	//start at beginning but OPTIMISE later
 	double onsetLikelihoodToNoise = 0.3;
 	
@@ -224,13 +228,13 @@
 	int numberOfMatchesFound = 0;
 
 	
-	double startTime = bayesianStruct.likelihood.offset;
-	double endTime = bayesianStruct.likelihood.offset + matchWindowWidth;
+	double startMatchingTime = bayesianStruct.likelihood.offset;
+	double endMatchingTime = bayesianStruct.likelihood.offset + matchWindowWidth;
 	
 	if (channel <= recordedTracks.numberOfAudioTracks){
 		for (int i = 0;i < recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){
 			double millisTime = recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].millisTime;
-			if (millisTime >= startTime && millisTime <= endTime){
+			if (millisTime >= startMatchingTime && millisTime <= endMatchingTime){
 				bayesianStruct.likelihood.addGaussianShapeFromRealTime(millisTime, likelihoodWidth,  quantity);
 				numberOfMatchesFound++;
 		//		printf("Adding Gaussian for onset at time %f offset %f\n", millisTime, bayesianStruct.likelihood.offset);
@@ -245,8 +249,8 @@
 	
 	bayesianStruct.calculatePosterior();
 	
-	lastAlignmentTime = ofGetElapsedTimeMillis();
-	recentEventTime[channel] = ofGetElapsedTimeMillis() - startTime;
+	lastAlignmentTime = timeIn;//use TIMESTAMP
+	recentEventTime[channel] = timeIn;//ofGetElapsedTimeMillis() - startTime;
 	
 }
 
@@ -254,8 +258,11 @@
 
 void AudioEventMatcher::matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn){
 	//start at beginning but OPTIMISE later
-	
-	
+	/*printf("TIME %i\n", ofGetElapsedTimeMillis());
+	//tmp debug
+	updateBestAlignmentPosition();
+	printf("current alignment best estimate %f\n", currentAlignmentPosition);
+	*/
 	bayesianStruct.updateBayesianDistributions(timeIn);//moves the posterior up into prior given the time interval and calculates new offsets
 
 	//set the lielihoods by matching the pitched note
@@ -295,8 +302,8 @@
 		bayesianStruct.calculatePosterior();
 	}
 	
-	lastAlignmentTime = ofGetElapsedTimeMillis();
-	recentEventTime[channel] = ofGetElapsedTimeMillis() - startTime;
+	lastAlignmentTime = timeIn;//has to use the STAMPED time
+	recentEventTime[channel] = timeIn;
 }
 
 double AudioEventMatcher::getPitchDistance(const double& pitchOne, const double& pitchTwo, const double& scale){
@@ -328,5 +335,13 @@
 	bayesPositionWindow.resized(w,h);
 }
 
+/*
+ 
+void printPosteriorMAPinfo(){	//tmp print stuff
+ printf("New pitch MAP post estimate now %i, ", bayesianStruct.posterior.MAPestimate);
+ double tmp  = bayesianStruct.posterior.getMAPestimate();
+ printf(" getting it %f and offset %f == %f ms\n", tmp,  bayesianStruct.posterior.offset, bayesianStruct.posterior.getIndexInRealTerms(tmp));
+ 
+ }
+ */
 
-
--- a/src/AudioEventMatcher.h	Fri Feb 03 17:53:14 2012 +0000
+++ b/src/AudioEventMatcher.h	Sat Feb 04 19:59:27 2012 +0000
@@ -65,12 +65,13 @@
 	
 	DynamicVector likelihoodVisualisation[3];
 	DynamicVector recentPrior;
+	DynamicVector projectedPrior;
 	
 	double currentAlignmentPosition;
 	double lastAlignmentTime;
 	
 	double recentEventTime[3];
-	double startTime;
+	int startTime;
 	
 	double screenStartTimeMillis, screenEndTimeMillis, screenWidthMillis;
 	bool followingLiveInput;
--- a/src/RecordedMultitrackAudio.cpp	Fri Feb 03 17:53:14 2012 +0000
+++ b/src/RecordedMultitrackAudio.cpp	Sat Feb 04 19:59:27 2012 +0000
@@ -13,7 +13,16 @@
 void RecordedMultitrackAudio::loadTestAudio(){
 	
 	
-	const char	*infilename = "../../../data/sound/LiveDues/kick_liveDues.wav";	
+//	const char	*kickfilename = "../../../data/sound/LiveDues/kick_liveDues.wav";	
+//	const char	*bassfilename = "../../../data/sound/LiveDues/bass_upsideLive.wav";	
+//	const char	*snarefilename = "../../../data/sound/LiveDues/snare_liveDues.wav";	
+	
+	const char	*kickfilename = "../../../data/sound/DiamondMatch1/kick_bip.wav";	
+	const char	*bassfilename = "../../../data/sound/DiamondMatch1/bass_bip.wav";	
+	const char	*snarefilename = "../../../data/sound/DiamondMatch1/snare_bip.wav";	
+	
+	
+	
 	//"../../../data/sound/basicClavScale.wav";	
 	
 	//LoadedAudioHolder lah;
@@ -26,7 +35,7 @@
 	//the above code using lah has problem that it deletes objects once out of the scope of testApp.setup()
 	//when lah is in theory no longer used - something like that possible? - at least pointers to onset detection seem deleted
 	loadedAudioPtr = new LoadedAudioHolder();
-	loadedAudioPtr->loadAudioFile(infilename);
+	loadedAudioPtr->loadAudioFile(kickfilename);
 	//	loadedAudioFiles.push_back(*loadedAudioPtr);
 	loadedAudioFiles[0] = *loadedAudioPtr;
 	
@@ -39,19 +48,17 @@
 	printf("BEFORE LOADING 1\n");
 	printInfo();
 	
-	infilename = "../../../data/sound/LiveDues/bass_upsideLive.wav";	
-	
 	loadedAudioPtr = new LoadedAudioHolder;
-	loadedAudioPtr->loadAudioFile(infilename);
+	loadedAudioPtr->loadAudioFile(bassfilename);
 	//	loadedAudioFiles.push_back(*loadedAudioPtr);
 	loadedAudioFiles[1] = *loadedAudioPtr;
 	loadedAudioFiles[1].fileLoader.onsetDetect.window.setToRelativeSize(0, trackScreenHeight, 1, trackScreenHeight);
 	loadedAudioFiles[1].setTrackType(1);
 
-	infilename = "../../../data/sound/LiveDues/snare_liveDues.wav";	
+	
 	
 	loadedAudioPtr = new LoadedAudioHolder;
-	loadedAudioPtr->loadAudioFile(infilename);
+	loadedAudioPtr->loadAudioFile(snarefilename);
 	//	loadedAudioFiles.push_back(*loadedAudioPtr);
 	loadedAudioFiles[2] = *loadedAudioPtr;
 	loadedAudioFiles[2].fileLoader.onsetDetect.window.setToRelativeSize(0, trackScreenHeight*2, 1, trackScreenHeight);
--- a/src/testApp.cpp	Fri Feb 03 17:53:14 2012 +0000
+++ b/src/testApp.cpp	Sat Feb 04 19:59:27 2012 +0000
@@ -66,19 +66,25 @@
 			int testChannel = m.getArgAsInt32(0);
 			double timeIn = m.getArgAsInt32(1); 
 			printf("\nKICK RECEIVED at time %f\n", timeIn);
+		
 			eventMatcher.newKickEvent(testChannel, timeIn);
+			
 		}
 		
 		if ( m.getAddress() == "/snare" ){
 			int testChannel = m.getArgAsInt32(0);
 			double timeIn = m.getArgAsInt32(1);
 			printf("\nSNARE RECEIVED at time %f\n", timeIn);
+			
 			eventMatcher.newSnareEvent(testChannel, timeIn);
+			
 		}
 		
 		if ( m.getAddress() == "/start" ){
 			printf("start!\n");
+			printf("STRART TIME IN %i\n", ofGetElapsedTimeMillis());
 			eventMatcher.startPlaying();
+			printf("TIME OUT %i\n", ofGetElapsedTimeMillis());
 		}
 	}
 }