Mercurial > hg > multitrack-audio-matcher

changeset 3:5e188c0035b6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
checking the offsets of the arrays
author Andrew N Robertson <andrew.robertson@eecs.qmul.ac.uk>
date Wed, 01 Feb 2012 16:05:26 +0000
parents 179c09199b3c
children 45b5cf9be377
files README.txt bayesianArraySrc/BayesianArrayStructure.cpp bayesianArraySrc/DynamicVector.cpp src/AudioEventMatcher.cpp src/AudioEventMatcher.h src/LiveAudioInput.cpp src/RecordedMultitrackAudio.cpp src/RecordedMultitrackAudio.h src/testApp.cpp
diffstat 9 files changed, 181 insertions(+), 89 deletions(-) [+]
line wrap: on
line diff
--- a/README.txt	Tue Jan 31 21:34:19 2012 +0000
+++ b/README.txt	Wed Feb 01 16:05:26 2012 +0000
@@ -15,3 +15,7 @@
 
 when adding gaussians - need to convert to the matrix indices - currently not done??
 
+Improve pitch distance by checking for octaves etc
+
+Examine yin and audio so that pitch algorithm is CORRECT!
+
--- a/bayesianArraySrc/BayesianArrayStructure.cpp	Tue Jan 31 21:34:19 2012 +0000
+++ b/bayesianArraySrc/BayesianArrayStructure.cpp	Wed Feb 01 16:05:26 2012 +0000
@@ -30,7 +30,7 @@
 
 	tmpBestEstimate = 0;
 	crossUpdateTimeThreshold = 60;
-	priorWidth = 50;
+	priorWidth = 20;
 	
 }
 
@@ -136,6 +136,7 @@
 	lastEventTime = 0;
 	bestEstimate = 0;
 	lastBestEstimateUpdateTime = 0;
+	
 	if (*realTimeMode)
 		lastBestEstimateUpdateTime = ofGetElapsedTimeMillis();
 	//cannot just be zero - offline bug
@@ -157,9 +158,12 @@
 	setNewDistributionOffsets(0);
 	
 	int zeroIndex = posterior.getRealTermsAsIndex(0);
+	printf("ZERO INDEX %i\n", zeroIndex);
 	
-	posterior.addGaussianShape(zeroIndex, 500, 1);
-	//	posterior.addToIndex(0, 1);
+	posterior.addGaussianShapeFromRealTime(0, 500, 1);//one way to add at x msec
+	posterior.addGaussianShape(posterior.getRealTermsAsIndex(10), 50, 1);//alternative way
+	
+	//posterior.addToIndex(0, 1);
 	likelihood.addConstant(1);
 	
 	updateCounter = 0;
@@ -168,6 +172,8 @@
 	printf("bayes reset arrays - best estimate %f\n", lastBestEstimateUpdateTime);
 	
 	setSpeedPrior(speedPriorValue);
+	relativeSpeedPosterior.copyFromDynamicVector(relativeSpeedPrior);
+	
 }
 
 
@@ -199,6 +205,8 @@
 	
 void BayesianArrayStructure::updateBestEstimate(const double& timeDifference){
 //	double timeDiff = ofGetElapsedTimeMillis() - lastEventTime;//
+	double tmp = bestEstimate;
+	printf("post offest %i\n", posterior.offset);
 	
 	double timeDiff = timeDifference;
 	
@@ -217,6 +225,8 @@
 	speedEstimate = relativeSpeedPosterior.getIndexInRealTerms(speedEstimateIndex);
 	bestEstimate = posterior.getIndexInRealTerms(posterior.MAPestimate) + timeDiff*speedEstimate;
 	
+	printf("best estimate update from %f to %f; time diff %f MAP %i = %f ms speed %f\n", tmp, bestEstimate, timeDiff,
+		   posterior.MAPestimate, posterior.getIndexInRealTerms(posterior.MAPestimate), speedEstimate);
 }
 
 void BayesianArrayStructure::calculatePosterior(){
--- a/bayesianArraySrc/DynamicVector.cpp	Tue Jan 31 21:34:19 2012 +0000
+++ b/bayesianArraySrc/DynamicVector.cpp	Wed Feb 01 16:05:26 2012 +0000
@@ -29,7 +29,9 @@
 }
 
 void DynamicVector::copyFromDynamicVector(const DynamicVector& dynamicVec){
+
 	if (dynamicVec.length == length){
+		offset = dynamicVec.offset;
 		for (int i = 0;i < length;i++)
 			array[i] = dynamicVec.array[i];
 	}
@@ -162,7 +164,7 @@
 void DynamicVector::addGaussianShapeFromRealTime(const double& actualTime, const double& StdDev, double factor){
 	
 	double mean = getRealTermsAsIndex(actualTime);
-	printf("Gaussian realtime %f at index %f\n", actualTime, mean);
+//	printf("Gaussian realtime %f at index %f\n", actualTime, mean);
 	int i;
 	double std_dev_factor = (2*StdDev*StdDev);
 	factor *= (1/(StdDev*sqrt(2*PI)));
@@ -277,7 +279,8 @@
 		int newHeightPixel;
 		for (int i = startInt;i < endInt;i++){
 			newHeightPixel = (int) heightConstant * (maxVal - array[i]);
-			ofLine (stepSize*(i-1), lastHeightPixel, stepSize*i, newHeightPixel);
+			int xPos = i - startInt;
+			ofLine (stepSize*(xPos-1), lastHeightPixel, stepSize*xPos, newHeightPixel);
 			lastHeightPixel = newHeightPixel;
 		}
 	
@@ -298,10 +301,11 @@
 	int newHeightPixel;
 	for (int i = startInt;i < endInt;i++){
 		newHeightPixel = (int) heightConstant * (maxVal - array[i]);
-		ofLine (window.x+stepSize*(i-1), window.y+lastHeightPixel, window.x+stepSize*i, window.y+newHeightPixel);
+		int xPos = i - startInt;
+		ofLine (window.x+stepSize*(xPos-1), window.y+lastHeightPixel, window.x+stepSize*xPos, window.y+newHeightPixel);
 		lastHeightPixel = newHeightPixel;
 	}
-	
+	ofDrawBitmapString("start index "+ofToString(startInt)+" end "+ofToString(endInt), window.x+20, window.y+20);
 }
 
 
--- a/src/AudioEventMatcher.cpp	Tue Jan 31 21:34:19 2012 +0000
+++ b/src/AudioEventMatcher.cpp	Wed Feb 01 16:05:26 2012 +0000
@@ -13,11 +13,14 @@
 const int matchWindowWidth = 6000;
 
 AudioEventMatcher::AudioEventMatcher(){
-
-	bayesTempoWindow.setToRelativeSize(0, 0.6, 1, 0.2);
-	bayesPositionWindow.setToRelativeSize(0, 0.8, 1, 0.2);
-
+	bayesPositionWindow.setToRelativeSize(0, 0.4, 1, 0.2);
+	bayesTempoWindow.setToRelativeSize(0, 0.8, 1, 0.2);
+	bayesLikelihoodWindow.setToRelativeSize(0, 0.6, 1, 0.2);
+	
 	setArraySizes();
+	
+	usingRealTime = false;
+	bayesianStruct.realTimeMode = &usingRealTime;
 }
 
 
@@ -32,8 +35,15 @@
 	
 }
 
+void  AudioEventMatcher::startPlaying(){
+	bayesianStruct.setStartPlaying();
+	//bayesianStruct.posterior.printArray();
+}
+
 void AudioEventMatcher::draw(){
-	//ofRect(20, 20, 300, 200);
+	ofSetColor(20,200,200);
+	bayesPositionWindow.drawOutline();
+	bayesTempoWindow.drawOutline();
 	
 	recordedTracks.drawTracks();
 	
@@ -42,18 +52,32 @@
 	
 	double screenWidthMillis = recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.framesToMillis(recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.amplitudeNumber);
 	
+	bayesianStruct.likelihood.drawVector(bayesianStruct.likelihood.getRealTermsAsIndex(0), bayesianStruct.likelihood.getRealTermsAsIndex(screenWidthMillis), bayesLikelihoodWindow);
 	
-	bayesianStruct.likelihood.drawVector(0, screenWidthMillis, bayesTempoWindow);
+	bayesianStruct.relativeSpeedPosterior.drawVector(0, bayesianStruct.relativeSpeedPosterior.getRealTermsAsIndex(2), bayesTempoWindow);
+	
+	bayesianStruct.posterior.drawVector(bayesianStruct.posterior.getRealTermsAsIndex(0), bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis), bayesPositionWindow);
+	
+	string tmpStr = "zero is "+ofToString(bayesianStruct.posterior.getRealTermsAsIndex(0));
+	tmpStr += " offsetis "+ofToString(bayesianStruct.posterior.offset);
+	tmpStr += " screenWidth = "+ofToString(bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis));
+	ofDrawBitmapString(tmpStr, 20,140);
+	tmpStr = "best est "+ofToString(bayesianStruct.bestEstimate);
+	ofDrawBitmapString(tmpStr, 20, 180);
+	
+	ofDrawBitmapString("screenamp "+ofToString(screenWidthMillis), 20, 100);
+	
 	
 }
 
 
 void AudioEventMatcher::newPitchEvent(const double& pitchIn, const double& timeIn){
 	liveInput.addPitchEvent(pitchIn, timeIn);
-	recordedTracks.matchNewPitchEvent(0, pitchIn, timeIn);
+	matchNewPitchEvent(0, pitchIn, timeIn);
 }
 
 void AudioEventMatcher::newKickEvent(const double& timeIn){
+//	liveInput.addKickEvent(time);
 	matchNewOnsetEvent(0, timeIn);
 }
 
@@ -64,8 +88,10 @@
 
 //Needs just to set bounds for the matching process, not have TimeIn
 void AudioEventMatcher::matchNewOnsetEvent(const int& channel, const double& timeIn){
+
+	
 	//start at beginning but OPTIMISE later
-	double likelihoodToNoise = 0.5;
+	double onsetLikelihoodToNoise = 0.5;
 	
 	double likelihoodWidth = 40;
 	
@@ -91,14 +117,112 @@
 		}
 	}
 	
+//	bayesianStruct.likelihood.addConstant((1-likelihoodToNoiseRatio)/bayesianStruct.likelihood.length);
+	bayesianStruct.likelihood.addConstant(numberOfMatchesFound*(1-onsetLikelihoodToNoise)/(onsetLikelihoodToNoise*bayesianStruct.likelihood.length));
+
 	bayesianStruct.likelihood.renormalise();
 	
-	//bayesStruct.likelihood.addConstant((1-likelihoodToNoiseRatio)/bayesStruct.likelihood.length);
+		
+}
+
+
+
+void AudioEventMatcher::matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn){
+	//start at beginning but OPTIMISE later
 	
 	
+	updateBayesianDistributions(timeIn);
+	
+	///set offsets
+//	bayesianStruct.likelihood.offset = bayesianStruct.prior.offset;
+	double pitchLikelihoodToNoise = 0.5;
+	int numberOfMatches = 0;
+	bayesianStruct.likelihood.zero();//set to zero
+	
+	double quantity = 0;
+	if (channel <= recordedTracks.numberOfAudioTracks){
+		for (int i = 0;i < recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){
+			
+			if (checkMatch(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn)) {
+				quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 40);
+				bayesianStruct.likelihood.addGaussianShapeFromRealTime(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].millisTime, 30, quantity);
+				recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = true;
+				numberOfMatches++;
+			} 
+			else{
+				recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = false;
+			}
+			
+		}
+	}
+	
+	bayesianStruct.likelihood.addConstant(numberOfMatches*(1-pitchLikelihoodToNoise)/(pitchLikelihoodToNoise*bayesianStruct.likelihood.length));
+	
+	recordedTracks.recentPitch = pitchIn;
 }
 
+double AudioEventMatcher::getPitchDistance(const double& pitchOne, const double& pitchTwo, const double& scale){
+	
+	double distance = abs(pitchOne - pitchTwo);
+	if (distance < scale)
+		distance = 1 - (distance/scale);
+	else
+		distance = 0;
+	
+//	printf("[pitch distance %f vs %f = %f\n", pitchOne, pitchTwo, distance);
+	return distance;
+
+}
+
+
+bool AudioEventMatcher::checkMatch(const double& recordedPitch, const double& livePitch){
+	if (abs(recordedPitch - livePitch) < 40)
+		return true;
+	else
+		return false;
+}
+
+
 
 void AudioEventMatcher::windowResized(const int& w, const int& h){
 	recordedTracks.windowResized(w,h);
-}
\ No newline at end of file
+	bayesTempoWindow.resized(w,h);
+	bayesPositionWindow.resized(w,h);
+}
+
+
+
+void  AudioEventMatcher::updateBayesianDistributions(const double& newEventTime){
+	//MOVE INTO bayesianStruct?? XX
+	
+
+	
+	//NEED TO CHECK HERE THAT THEY HAVE THE SAME OFFSETS
+	bayesianStruct.prior.copyFromDynamicVector(bayesianStruct.posterior);//try the otehr way
+
+	//bayesianStruct.copyPriorToPosterior();
+	//need to get new MAP position and set the offset of the arrays
+	//currently bestEstimate is the approx for the new MAP position
+		
+	double timeDifference = newEventTime - bayesianStruct.lastEventTime; 
+	printf("updating distributions at time %f diff %f\n", newEventTime, timeDifference);
+	
+	//addnoise to the tempo distribution
+	//bayesianStruct.decaySpeedDistribution(timeDifference);
+	
+	if (timeDifference > 50){
+		bayesianStruct.addGaussianNoiseToSpeedPosterior(timeDifference * 10.0 / 100.);
+	}
+	
+	bayesianStruct.updateBestEstimate(timeDifference);
+	
+	bayesianStruct.lastBestEstimateUpdateTime = newEventTime;//getTimeNow(timePlayed);
+		
+	bayesianStruct.setNewDistributionOffsets(max(0., bayesianStruct.bestEstimate - (bayesianStruct.prior.scalar*bayesianStruct.prior.arraySize/2)));
+	
+	bayesianStruct.crossUpdateArrays(bayesianStruct.posterior, bayesianStruct.relativeSpeedPosterior, timeDifference);
+	
+	bayesianStruct.posterior.offset = max(0., bayesianStruct.bestEstimate - (bayesianStruct.prior.scalar*bayesianStruct.prior.arraySize/2));//	bayesianStruct.prior.offset = max(0., bayesianStruct.bestEstimate - (bayesianStruct.prior.scalar*bayesianStruct.prior.arraySize/2));
+		
+	bayesianStruct.lastEventTime = newEventTime;//bayesianStruct.lastEventTime = ofGetElapsedTimeMillis();
+}
--- a/src/AudioEventMatcher.h	Tue Jan 31 21:34:19 2012 +0000
+++ b/src/AudioEventMatcher.h	Wed Feb 01 16:05:26 2012 +0000
@@ -44,8 +44,16 @@
 	void windowResized(const int& w, const int& h);
 	
 	ofxWindowRegion bayesTempoWindow;
-	ofxWindowRegion bayesPositionWindow;	
+	ofxWindowRegion bayesPositionWindow;
+	ofxWindowRegion bayesLikelihoodWindow;
 
-
+	bool checkMatch(const double& recordedPitch, const double& livePitch);
+	double getPitchDistance(const double& pitchOne, const double& pitchTwo, const double& scale);
+	
+	void updateBayesianDistributions(const double& newEventTime);
+	void startPlaying();
+	
+	bool usingRealTime;
+	
 };
 #endif
--- a/src/LiveAudioInput.cpp	Tue Jan 31 21:34:19 2012 +0000
+++ b/src/LiveAudioInput.cpp	Wed Feb 01 16:05:26 2012 +0000
@@ -26,7 +26,7 @@
 	e.frameTime = millisToFrames(time);
 	e.pitch = pitch;
 	liveEvents.push_back(e);
-	printf("live input pitch %f time %f ms == %f frames\n", pitch, time, e.frameTime);
+	//printf("live input pitch %f time %f ms == %f frames\n", pitch, time, e.frameTime);
 	numberOfEvents++;
 	
 }
--- a/src/RecordedMultitrackAudio.cpp	Tue Jan 31 21:34:19 2012 +0000
+++ b/src/RecordedMultitrackAudio.cpp	Wed Feb 01 16:05:26 2012 +0000
@@ -28,7 +28,7 @@
 	//	loadedAudioFiles.push_back(*loadedAudioPtr);
 	loadedAudioFiles[0] = *loadedAudioPtr;
 	
-	loadedAudioFiles[0].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.0, 1, 0.25);
+	loadedAudioFiles[0].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.0, 1, 0.2);
 	loadedAudioFiles[0].setTrackType(0);//	fileLoader.onsetDetect.trackType = 0;
 	
 	//	printf("Loaded audio %i\n", (int)numberOfAudioTracks);
@@ -43,7 +43,7 @@
 	loadedAudioPtr->loadAudioFile(infilename);
 	//	loadedAudioFiles.push_back(*loadedAudioPtr);
 	loadedAudioFiles[1] = *loadedAudioPtr;
-	loadedAudioFiles[1].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.3, 1, 0.25);
+	loadedAudioFiles[1].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.2, 1, 0.2);
 	loadedAudioFiles[1].setTrackType(1);
 //	loadedAudioFiles[1].fileLoader.onsetDetect.trackType = 0;
 	
@@ -73,32 +73,6 @@
 }
 
 
-void RecordedMultitrackAudio::matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn){
-	//start at beginning but OPTIMISE later
-	
-	if (channel <= numberOfAudioTracks){
-		for (int i = 0;i < loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){
-			
-			if (checkMatch(loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn)) {
-				 loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = true;
-			 } 
-			 else{
-				 loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = false;
-			 }
-			
-		 }
-	}
-	
-	recentPitch = pitchIn;
-}
-
-bool RecordedMultitrackAudio::checkMatch(const double& recordedPitch, const double& livePitch){
-	if (abs(recordedPitch - livePitch) < 40)
-		return true;
-	else
-		return false;
-}
-
 
 void RecordedMultitrackAudio::switchScreens(){
 	for (int i = 0;i < numberOfAudioTracks;i++)
--- a/src/RecordedMultitrackAudio.h	Tue Jan 31 21:34:19 2012 +0000
+++ b/src/RecordedMultitrackAudio.h	Wed Feb 01 16:05:26 2012 +0000
@@ -43,9 +43,9 @@
 	void zoomOut();
 	
 	
-	void matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn);
+//	void matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn);
 
-	bool checkMatch(const double& recordedPitch, const double& livePitch);
+//	bool checkMatch(const double& recordedPitch, const double& livePitch);
 	
 	double recentPitch;
 	//variables
--- a/src/testApp.cpp	Tue Jan 31 21:34:19 2012 +0000
+++ b/src/testApp.cpp	Wed Feb 01 16:05:26 2012 +0000
@@ -34,44 +34,7 @@
 	
 	//audioFilePlayer.loadAudioFile(infilename);
 }
-/*
-void testApp::loadTestAudio(){
-	
 
-	const char	*infilename = "../../../data/sound/bach4_short1.wav";	
-	
-	//LoadedAudioHolder lah;
-	//	lah.loadAudioFile(infilename);
-	//	loadedAudioFiles.push_back(lah);
-	
-	//Take care here - we need a pointer to create new instance
-	//but not then delete the instance before the vector of all audio tracks has been used
-	//the above code using lah has problem that it deletes objects once out of the scope of testApp.setup()
-	//when lah is in theory no longer used - something like that possible? - at least pointers to onset detection seem deleted
-	loadedAudioPtr = new LoadedAudioHolder;
-	loadedAudioPtr->loadAudioFile(infilename);
-	//	loadedAudioFiles.push_back(*loadedAudioPtr);
-	loadedAudioFiles[0] = *loadedAudioPtr;
-	
-	loadedAudioFiles[0].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.0, 1, 0.25);
-	
-	//	printf("Loaded audio %i\n", (int)numberOfAudioTracks);
-	printf("loaded max val  is %f\n", loadedAudioFiles[0].fileLoader.onsetDetect.onsetDetector.maximumDetectionValue);
-	
-	printf("BEFORE LOADING 1\n");
-	keyPressed('p');
-	
-	loadedAudioPtr = new LoadedAudioHolder;
-	loadedAudioPtr->loadAudioFile(infilename);
-	//	loadedAudioFiles.push_back(*loadedAudioPtr);
-	loadedAudioFiles[1] = *loadedAudioPtr;
-	loadedAudioFiles[1].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.3, 1, 0.25);
-
-	printf("AFTER LOADING 1\n");
-	keyPressed('p');
-	
-}
- */
 
 //--------------------------------------------------------------
 void testApp::update(){
@@ -96,7 +59,7 @@
 		if ( m.getAddress() == "/aubioPitch" ){
 			float pitchIn = m.getArgAsFloat(0); 
 			int timeIn = m.getArgAsInt32(1); 
-			printf("AUBIO PITCH RECEIVED %f at time %i\n", pitchIn, timeIn);
+			printf("\nAUBIO PITCH RECEIVED %f at time %i\n", pitchIn, timeIn);
 			eventMatcher.newPitchEvent(pitchIn, timeIn);
 		}
 		
@@ -113,6 +76,11 @@
 			printf("snare RECEIVED at time %f\n", timeIn);
 			eventMatcher.newSnareEvent(timeIn);
 		}
+		
+		if ( m.getAddress() == "/start" ){
+			printf("start!\n");
+			eventMatcher.startPlaying();
+		}
 	}
 }
 
@@ -170,7 +138,7 @@
 	}
 
 	if (key == 'p'){
-
+		eventMatcher.bayesianStruct.posterior.printArray();
 	}