Mercurial > hg > multitrack-audio-matcher

changeset 7:33dedfe32893

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
kick, snare and bass windowed. Likelihoods in dedicated screen regions
author Andrew N Robertson <andrew.robertson@eecs.qmul.ac.uk>
date Thu, 02 Feb 2012 21:55:51 +0000
parents 746a5af43c02
children 572564b7cb85
files bayesianArraySrc/BayesianArrayStructure.cpp bayesianArraySrc/BayesianArrayStructure.h src/AudioEventMatcher.cpp src/AudioEventMatcher.h src/RecordedMultitrackAudio.cpp src/RecordedMultitrackAudio.h src/testApp.cpp
diffstat 7 files changed, 82 insertions(+), 28 deletions(-) [+]
line wrap: on
line diff
--- a/bayesianArraySrc/BayesianArrayStructure.cpp	Thu Feb 02 17:52:08 2012 +0000
+++ b/bayesianArraySrc/BayesianArrayStructure.cpp	Thu Feb 02 21:55:51 2012 +0000
@@ -16,6 +16,7 @@
 	printf("Bayesian structure: DeFault constructor called");
 
 	usingIntegratedTempoEstimate = false;// use max index
+	updatingSpeedDistribution = false;
 	
 	relativeSpeedLikelihoodStdDev = 5.0;
 	
@@ -104,7 +105,8 @@
 	relativeSpeedPosterior.zero();
 	relativeSpeedPosterior.addGaussianShape(index, priorWidth, 0.8);
 	printf("speed adding to index for 1 = %f\n", relativeSpeedPosterior.getRealTermsAsIndex(1));
-	relativeSpeedPosterior.addToIndex(relativeSpeedPosterior.getRealTermsAsIndex(1), 1);
+	relativeSpeedPosterior.addToIndex(relativeSpeedPosterior.getRealTermsAsIndex(1), 0.1);
+	relativeSpeedPosterior.addGaussianShapeFromRealTime(1, 3, 0.5);
 	
 	relativeSpeedPosterior.renormalise();
 	relativeSpeedPosterior.getMaximum();
@@ -163,8 +165,8 @@
 	int zeroIndex = posterior.getRealTermsAsIndex(0);
 	printf("ZERO INDEX %i\n", zeroIndex);
 	
-	posterior.addGaussianShapeFromRealTime(0, 500, 1);//one way to add at x msec
-	posterior.addGaussianShape(posterior.getRealTermsAsIndex(10), 50, 1);//alternative way
+	posterior.addGaussianShapeFromRealTime(0, 60, 1);//one way to add at x msec
+//	posterior.addGaussianShape(posterior.getRealTermsAsIndex(10), 50, 1);//alternative way
 	
 	//posterior.addToIndex(0, 1);
 	likelihood.addConstant(1);
@@ -288,7 +290,7 @@
 	//addnoise to the tempo distribution
 	//bayesianStruct.decaySpeedDistribution(timeDifference);
 	
-	if (timeDifference > 50){
+	if (timeDifference > 50 && updatingSpeedDistribution){
 		addGaussianNoiseToSpeedPosterior(timeDifference * 10.0 / 100.);
 	}
 
--- a/bayesianArraySrc/BayesianArrayStructure.h	Thu Feb 02 17:52:08 2012 +0000
+++ b/bayesianArraySrc/BayesianArrayStructure.h	Thu Feb 02 21:55:51 2012 +0000
@@ -99,6 +99,7 @@
 	double relativeSpeedLikelihoodStdDev;
 	
 	void printPostOffset();
+	bool updatingSpeedDistribution;//false for testing
 
 };
 #endif
--- a/src/AudioEventMatcher.cpp	Thu Feb 02 17:52:08 2012 +0000
+++ b/src/AudioEventMatcher.cpp	Thu Feb 02 21:55:51 2012 +0000
@@ -13,16 +13,25 @@
 const int matchWindowWidth = 6000;
 
 AudioEventMatcher::AudioEventMatcher(){
-	bayesPositionWindow.setToRelativeSize(0, 0.4, 1, 0.2);
-	bayesTempoWindow.setToRelativeSize(0, 0.8, 1, 0.2);
-	bayesLikelihoodWindow.setToRelativeSize(0, 0.6, 1, 0.2);
-	
+
 	setArraySizes();
 	
 	usingRealTime = false;
 	bayesianStruct.realTimeMode = &usingRealTime;
+	recentPitch = 0;
 }
 
+void AudioEventMatcher::setWindowDimensions(){
+	double startHeight = recordedTracks.numberOfAudioTracks * recordedTracks.trackScreenHeight;
+	double heightAvailable = 1 - startHeight;
+	heightAvailable /= 3.0;
+	
+	bayesPositionWindow.setToRelativeSize(0, startHeight, 1, heightAvailable);
+	bayesLikelihoodWindow.setToRelativeSize(0, startHeight + 1*heightAvailable, 1, heightAvailable);
+	bayesTempoWindow.setToRelativeSize(0, startHeight + 2*heightAvailable, 1, heightAvailable);
+	
+	
+}
 
 void AudioEventMatcher::setArraySizes(){
 	bayesianStruct.resetSpeedSize(200);
@@ -48,6 +57,8 @@
 	
 	//draw the scrolling audio tracks
 	recordedTracks.drawTracks();
+
+
 	
 	ofSetColor(255);
 //	bayesianStruct.relativeSpeedPrior.drawVector(0, 200, bayesTempoWindow);
@@ -59,6 +70,7 @@
 	
 	//	bayesianStruct.relativeSpeedPosterior.drawVector(0, bayesianStruct.relativeSpeedPosterior.getRealTermsAsIndex(2), bayesTempoWindow);
 	
+	ofDrawBitmapString("pitch "+ofToString(recentPitch, 2)+", Time "+ofToString(recentTime, 0), 20, 20);
 }
 	
 void AudioEventMatcher::drawBayesianDistributions(){
@@ -96,9 +108,17 @@
 	ofDrawBitmapString("screenamp "+ofToString(screenWidthMillis), 20, 100);
 	
 	
+	
+	//draw track by track likelihoods
+	for (int i = 0; i <recordedTracks.numberOfAudioTracks;i++){
+		ofSetColor(200,255,50);
+		likelihoodVisualisation[i].drawConstrainedVector(startIndex, endIndex, 0, ofGetWidth(), recordedTracks.loadedAudioFiles[i].fileLoader.onsetDetect.window);		
+	}
+	
 }
 
 void AudioEventMatcher::newPitchEvent(const int& channel, const double& pitchIn, const double& timeIn){
+	if (pitchIn > 0){
 	liveInput.addPitchEvent(pitchIn, timeIn);
 	
 	//tmp print stuff
@@ -106,22 +126,37 @@
 	double tmp  = bayesianStruct.posterior.getMAPestimate();
 	printf(" getting it %f and offset %f == %f ms\n", tmp,  bayesianStruct.posterior.offset, bayesianStruct.posterior.getIndexInRealTerms(tmp));
 	
-	matchNewPitchEvent(channel, pitchIn, timeIn);
+	matchNewPitchEvent(channel, pitchIn, timeIn);//main pitch matching fn
+
+	likelihoodVisualisation[1] = bayesianStruct.likelihood;
+		
+	recentPitch = pitchIn;//for drawing
+	recentTime = timeIn;
+	}
 }
 
 void AudioEventMatcher::newKickEvent(const double& timeIn){	
 //	liveInput.addKickEvent(timeIn);
 	matchNewOnsetEvent(0, timeIn);
+	likelihoodVisualisation[0] = bayesianStruct.likelihood;
 }
 
 void AudioEventMatcher::newKickEvent(const int& channel, const double& timeIn){
 //	liveInput.addKickEvent(timeIn);
 	matchNewOnsetEvent(channel, timeIn);
+	likelihoodVisualisation[0] = bayesianStruct.likelihood;
 }
 
 
 void AudioEventMatcher::newSnareEvent(const double& timeIn){
 	matchNewOnsetEvent(2, timeIn);
+	likelihoodVisualisation[2] = bayesianStruct.likelihood;
+}
+
+
+void AudioEventMatcher::newSnareEvent(const int& channel, const double& timeIn){
+	matchNewOnsetEvent(channel, timeIn);
+	likelihoodVisualisation[2] = bayesianStruct.likelihood;
 }
 
 //Needs just to set bounds for the matching process, not have TimeIn
@@ -174,9 +209,9 @@
 	
 	bayesianStruct.updateBayesianDistributions(timeIn);//moves the posterior up into prior given the time interval and calculates new offsets
 	
-	///set offsets
-//	bayesianStruct.likelihood.offset = bayesianStruct.prior.offset;
-	double pitchLikelihoodToNoise = 0.2;//more noise
+	//set the lielihoods by matching the pitched note
+	
+	double pitchLikelihoodToNoise = 0.6;//more noise
 	int numberOfMatches = 0;
 	bayesianStruct.likelihood.zero();//set to zero
 	
@@ -185,7 +220,7 @@
 		for (int i = 0;i < recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){
 			
 			if (checkMatch(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn)) {
-				quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 20);
+				quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 10);
 				bayesianStruct.likelihood.addGaussianShapeFromRealTime(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].millisTime, 30, quantity);
 				recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = true;
 				numberOfMatches++;
@@ -198,12 +233,13 @@
 	}
 
 	if (numberOfMatches > 0){//no point updating unless there is a match
+	
 		bayesianStruct.likelihood.addConstant(numberOfMatches*(1-pitchLikelihoodToNoise)/(pitchLikelihoodToNoise*bayesianStruct.likelihood.length));
 
 	//tmp set likelihood constant and calculate using that
 	//bayesianStruct.likelihood.zero();
 	//bayesianStruct.likelihood.addConstant(1);
-	
+		
 		bayesianStruct.calculatePosterior();
 	}
 	
--- a/src/AudioEventMatcher.h	Thu Feb 02 17:52:08 2012 +0000
+++ b/src/AudioEventMatcher.h	Thu Feb 02 21:55:51 2012 +0000
@@ -19,6 +19,7 @@
 #include "ofxWindowRegion.h"
 #include "BayesianArrayStructure.h"
 #include "RecordedMultitrackAudio.h"
+#include "DynamicVector.h"
 
 class AudioEventMatcher{
 	
@@ -29,11 +30,13 @@
 	
 	void draw();
 	void drawBayesianDistributions();;
+	void setWindowDimensions();
 	
 	void newPitchEvent(const int& channel, const double& pitchIn, const double& timeIn);
 	void newKickEvent(const double& timeIn);
 	void newKickEvent(const int& channel, const double& timeIn);
 	void newSnareEvent(const double& timeIn);
+	void newSnareEvent(const int& channel, const double& timeIn);
 	
 	void matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn);
 	void matchNewOnsetEvent(const int& channel, const double& timeIn);
@@ -55,6 +58,10 @@
 	void startPlaying();
 	
 	bool usingRealTime;
+	double recentPitch, recentTime;
+	
+	DynamicVector likelihoodVisualisation[3];
+	DynamicVector recentPrior;
 
 };
 #endif
--- a/src/RecordedMultitrackAudio.cpp	Thu Feb 02 17:52:08 2012 +0000
+++ b/src/RecordedMultitrackAudio.cpp	Thu Feb 02 21:55:51 2012 +0000
@@ -13,13 +13,14 @@
 void RecordedMultitrackAudio::loadTestAudio(){
 	
 	
-	const char	*infilename = "../../../data/sound/LiveDues/neuRoom_liveDues.wav";	
+	const char	*infilename = "../../../data/sound/LiveDues/kick_liveDues.wav";	
 	//"../../../data/sound/basicClavScale.wav";	
 	
 	//LoadedAudioHolder lah;
 	//	lah.loadAudioFile(infilename);
 	//	loadedAudioFiles.push_back(lah);
 	
+	trackScreenHeight = 0.15;
 	//Take care here - we need a pointer to create new instance
 	//but not then delete the instance before the vector of all audio tracks has been used
 	//the above code using lah has problem that it deletes objects once out of the scope of testApp.setup()
@@ -29,7 +30,7 @@
 	//	loadedAudioFiles.push_back(*loadedAudioPtr);
 	loadedAudioFiles[0] = *loadedAudioPtr;
 	
-	loadedAudioFiles[0].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.0, 1, 0.2);
+	loadedAudioFiles[0].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.0, 1, trackScreenHeight);
 	loadedAudioFiles[0].setTrackType(0);//	fileLoader.onsetDetect.trackType = 0;
 	
 	//	printf("Loaded audio %i\n", (int)numberOfAudioTracks);
@@ -44,14 +45,23 @@
 	loadedAudioPtr->loadAudioFile(infilename);
 	//	loadedAudioFiles.push_back(*loadedAudioPtr);
 	loadedAudioFiles[1] = *loadedAudioPtr;
-	loadedAudioFiles[1].fileLoader.onsetDetect.window.setToRelativeSize(0, 0.2, 1, 0.2);
+	loadedAudioFiles[1].fileLoader.onsetDetect.window.setToRelativeSize(0, trackScreenHeight, 1, trackScreenHeight);
 	loadedAudioFiles[1].setTrackType(1);
-//	loadedAudioFiles[1].fileLoader.onsetDetect.trackType = 0;
+
+	infilename = "../../../data/sound/LiveDues/snare_liveDues.wav";	
+	
+	loadedAudioPtr = new LoadedAudioHolder;
+	loadedAudioPtr->loadAudioFile(infilename);
+	//	loadedAudioFiles.push_back(*loadedAudioPtr);
+	loadedAudioFiles[2] = *loadedAudioPtr;
+	loadedAudioFiles[2].fileLoader.onsetDetect.window.setToRelativeSize(0, trackScreenHeight*2, 1, trackScreenHeight);
+	loadedAudioFiles[2].setTrackType(2);
+	
 	
 	printf("AFTER LOADING 1\n");
 	printInfo();
 	
-	numberOfAudioTracks = 2;
+	numberOfAudioTracks = 3;
 	
 	
 }
@@ -63,8 +73,6 @@
 	for (int i = 0;i < numberOfAudioTracks;i++){		
 		loadedAudioFiles[i].draw();
 	}
-	
-	ofDrawBitmapString("pitch "+ofToString(recentPitch, 2), 20, 20);
 }
 
 
--- a/src/RecordedMultitrackAudio.h	Thu Feb 02 17:52:08 2012 +0000
+++ b/src/RecordedMultitrackAudio.h	Thu Feb 02 21:55:51 2012 +0000
@@ -47,12 +47,12 @@
 
 //	bool checkMatch(const double& recordedPitch, const double& livePitch);
 	
-	double recentPitch;
 	//variables
 	int numberOfAudioTracks;
 	LoadedAudioHolder* loadedAudioPtr;
 	LoadedAudioHolder loadedAudioFiles[5];
 	
+	double trackScreenHeight;
 	
 };
 #endif
\ No newline at end of file
--- a/src/testApp.cpp	Thu Feb 02 17:52:08 2012 +0000
+++ b/src/testApp.cpp	Thu Feb 02 21:55:51 2012 +0000
@@ -31,7 +31,7 @@
 	
 	eventMatcher.recordedTracks.loadTestAudio();
 	
-	
+	eventMatcher.setWindowDimensions();
 	//audioFilePlayer.loadAudioFile(infilename);
 }
 
@@ -66,15 +66,15 @@
 		//	float pitchIn = m.getArgAsFloat(0);
 			int testChannel = m.getArgAsInt32(0);
 			double timeIn = m.getArgAsInt32(1); 
-			printf("kick RECEIVED at time %f\n", timeIn);
+			printf("\nKICK RECEIVED at time %f\n", timeIn);
 			eventMatcher.newKickEvent(testChannel, timeIn);
 		}
 		
 		if ( m.getAddress() == "/snare" ){
-			//	float pitchIn = m.getArgAsFloat(0); 
-			double timeIn = m.getArgAsInt32(0); 
-			printf("snare RECEIVED at time %f\n", timeIn);
-			eventMatcher.newSnareEvent(timeIn);
+			int testChannel = m.getArgAsInt32(0);
+			double timeIn = m.getArgAsInt32(1);
+			printf("\nSNARE RECEIVED at time %f\n", timeIn);
+			eventMatcher.newSnareEvent(testChannel, timeIn);
 		}
 		
 		if ( m.getAddress() == "/start" ){