Mercurial > hg > multitrack-audio-matcher
changeset 6:746a5af43c02
windowed bayesian distributions - drawn within constrained portion of the screen
author | Andrew N Robertson <andrew.robertson@eecs.qmul.ac.uk> |
---|---|
date | Thu, 02 Feb 2012 17:52:08 +0000 |
parents | 5ef00d1dfe68 |
children | 33dedfe32893 |
files | bayesianArraySrc/BayesianArrayStructure.cpp bayesianArraySrc/BayesianArrayStructure.h bayesianArraySrc/DynamicVector.cpp bayesianArraySrc/DynamicVector.h src/AudioEventMatcher.cpp src/AudioEventMatcher.h src/RecordedMultitrackAudio.cpp src/testApp.cpp |
diffstat | 8 files changed, 154 insertions(+), 93 deletions(-) [+] |
line wrap: on
line diff
--- a/bayesianArraySrc/BayesianArrayStructure.cpp Thu Feb 02 12:45:36 2012 +0000 +++ b/bayesianArraySrc/BayesianArrayStructure.cpp Thu Feb 02 17:52:08 2012 +0000 @@ -246,6 +246,10 @@ posterior.renormalise(); + //tmp print stuff + printf("After CALC"); + printPostOffset(); + } @@ -268,6 +272,46 @@ } +void BayesianArrayStructure::updateBayesianDistributions(const double& newEventTime){ + + //NEED TO CHECK HERE THAT THEY HAVE THE SAME OFFSETS + prior.copyFromDynamicVector(posterior);//try the otehr way + + //bayesianStruct.copyPriorToPosterior(); + //need to get new MAP position and set the offset of the arrays + //currently bestEstimate is the approx for the new MAP position +// int tmpMap = bayesianStruct.posterior.getMAPestimate(); + + double timeDifference = newEventTime - lastEventTime; +// printf("updating distributions at time %f diff %f offset %f tmpmap est %i\n", newEventTime, timeDifference, bayesianStruct.posterior.offset, tmpMap); + + //addnoise to the tempo distribution + //bayesianStruct.decaySpeedDistribution(timeDifference); + + if (timeDifference > 50){ + addGaussianNoiseToSpeedPosterior(timeDifference * 10.0 / 100.); + } + + printPostOffset(); + + updateBestEstimate(timeDifference); + lastBestEstimateUpdateTime = newEventTime;//getTimeNow(timePlayed); + + + setNewDistributionOffsets(max(0., bestEstimate - (prior.scalar*prior.arraySize/2))); + crossUpdateArrays(posterior, relativeSpeedPosterior, timeDifference); + + //i.e. using the same offset as prior + posterior.offset = prior.offset;// + + // float tmpPrior = max(0., bestEstimate - (prior.scalar*prior.arraySize/2));// prior.offset = max(0., bestEstimate - (prior.scalar*prior.arraySize/2)); + // printf("Using prior offset of %f not %f\n", tmpPrior, prior.offset); + + lastEventTime = newEventTime;//lastEventTime = ofGetElapsedTimeMillis(); + +} + + void BayesianArrayStructure::crossUpdateArrays(DynamicVector& position, DynamicVector& speed, double timeDifference){ //set the cutoff for offset of position first! XXX @@ -602,6 +646,11 @@ } +void BayesianArrayStructure::printPostOffset(){ + double tmp = posterior.getMAPestimate(); + printf(" MAP index %i post offset %f == %f ms\n", posterior.MAPestimate, posterior.offset, posterior.getIndexInRealTerms(posterior.MAPestimate)); +} + /* void BayesianArrayStructure::updateTempoDistribution(const double& speedRatio, const double& matchFactor){
--- a/bayesianArraySrc/BayesianArrayStructure.h Thu Feb 02 12:45:36 2012 +0000 +++ b/bayesianArraySrc/BayesianArrayStructure.h Thu Feb 02 17:52:08 2012 +0000 @@ -86,6 +86,7 @@ void calculateTempoUpdate(); + void updateBayesianDistributions(const double& newEventTime); void crossUpdateArrays(DynamicVector& position, DynamicVector& speed, double timeDifference); void complexCrossUpdate(const double& timeDifferenceInPositionVectorUnits); void translateByMaximumSpeed(const double& timeDifferenceInPositionVectorUnits); @@ -96,6 +97,8 @@ bool* realTimeMode; bool usingIntegratedTempoEstimate; double relativeSpeedLikelihoodStdDev; + + void printPostOffset(); }; #endif
--- a/bayesianArraySrc/DynamicVector.cpp Thu Feb 02 12:45:36 2012 +0000 +++ b/bayesianArraySrc/DynamicVector.cpp Thu Feb 02 17:52:08 2012 +0000 @@ -328,11 +328,11 @@ double stepSize = (maxScreenIndex - minScreenIndex) / (double)(maxIndex - minIndex);//step size in pixels per array bin double screenHeight = ofGetHeight(); double maxVal = getMaximum(); - + //OPTIMIZE!! XXX could just add stepsize each time //not add minindex each time int i = max(1,minIndex+1); -// ofDrawBitmapString("i = "+ofToString(i)+" :: screen min: "+ofToString(minScreenIndex + stepSize*(i-minIndex-1)), 20, 640); + // ofDrawBitmapString("i = "+ofToString(i)+" :: screen min: "+ofToString(minScreenIndex + stepSize*(i-minIndex-1)), 20, 640); while ((minScreenIndex + stepSize*(i-minIndex)) < 0) i++;//only draw what is on the screen @@ -346,6 +346,36 @@ ofLine(minScreenIndex, screenHeight, minScreenIndex, screenHeight/2); ofLine(maxScreenIndex, screenHeight, maxScreenIndex, screenHeight/2); + // ofDrawBitmapString(ofToString(stepSize, 2)+" "+ofToString(maxScreenIndex - minScreenIndex, 0), 20, 600); + +} + + +void DynamicVector::drawConstrainedVector(const int& minIndex, const int& maxIndex, const int& minScreenIndex, const int& maxScreenIndex, const ofxWindowRegion& window){ + //constrain the height and width + + double stepSize = (maxScreenIndex - minScreenIndex) / (double)(maxIndex - minIndex);//step size in pixels per array bin + double screenHeight = window.height; + double maxVal = getMaximum(); + + //OPTIMIZE!! XXX could just add stepsize each time + //not add minindex each time + int i = max(1,minIndex+1); +// ofDrawBitmapString("i = "+ofToString(i)+" :: screen min: "+ofToString(minScreenIndex + stepSize*(i-minIndex-1)), 20, 640); + + while ((minScreenIndex + stepSize*(i-minIndex)) < 0) + i++;//only draw what is on the screen + + for ( ; i < min(maxIndex+1, (int)array.size());i++){ + ofLine (window.x+minScreenIndex + (stepSize*(i-minIndex-1)), window.y + screenHeight * (1 - array[i-1] / maxVal), + window.x + minScreenIndex + (stepSize*(i-minIndex)), window.y + screenHeight * (1 - array[i] / maxVal) ); + + } + + //some lines where the bounaries are + ofLine(window.x + minScreenIndex, window.y + screenHeight, window.x + minScreenIndex, window.y + screenHeight/2); + ofLine(window.x + maxScreenIndex, window.y + screenHeight, window.x + maxScreenIndex, window.y + screenHeight/2); + // ofDrawBitmapString(ofToString(stepSize, 2)+" "+ofToString(maxScreenIndex - minScreenIndex, 0), 20, 600); }
--- a/bayesianArraySrc/DynamicVector.h Thu Feb 02 12:45:36 2012 +0000 +++ b/bayesianArraySrc/DynamicVector.h Thu Feb 02 17:52:08 2012 +0000 @@ -41,6 +41,7 @@ void drawVector(const int& minIndex, const int& maxIndex, ofxWindowRegion window); void drawConstrainedVector(const int& minIndex, const int& maxIndex, const int& minScreenIndex, const int& maxScreenIndex); + void drawConstrainedVector(const int& minIndex, const int& maxIndex, const int& minScreenIndex, const int& maxScreenIndex, const ofxWindowRegion& window); void addConstant(const double& value); void addGaussianShape(const double& mean, const double& stddev, double factor);
--- a/src/AudioEventMatcher.cpp Thu Feb 02 12:45:36 2012 +0000 +++ b/src/AudioEventMatcher.cpp Thu Feb 02 17:52:08 2012 +0000 @@ -41,27 +41,51 @@ } void AudioEventMatcher::draw(){ + //draw some outlines in blue ofSetColor(20,200,200); bayesPositionWindow.drawOutline(); bayesTempoWindow.drawOutline(); + //draw the scrolling audio tracks recordedTracks.drawTracks(); ofSetColor(255); // bayesianStruct.relativeSpeedPrior.drawVector(0, 200, bayesTempoWindow); + drawBayesianDistributions(); + // bayesianStruct.posterior.drawVector(0, bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis), bayesPositionWindow); + + //bayesianStruct.posterior.drawVector(bayesianStruct.posterior.getRealTermsAsIndex(0), bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis), bayesPositionWindow); + + // bayesianStruct.relativeSpeedPosterior.drawVector(0, bayesianStruct.relativeSpeedPosterior.getRealTermsAsIndex(2), bayesTempoWindow); + +} + +void AudioEventMatcher::drawBayesianDistributions(){ + + double screenWidthMillis = recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.framesToMillis(recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.amplitudeNumber); - bayesianStruct.posterior.drawVector(0, bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis), bayesPositionWindow); + + double screenStartTimeMillis = recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.framesToMillis(recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.drawParams.windowStartFrame); + double screenEndTimeMillis = screenStartTimeMillis + screenWidthMillis; + int startIndex = bayesianStruct.posterior.getRealTermsAsIndex(screenStartTimeMillis); + int endIndex = bayesianStruct.posterior.getRealTermsAsIndex(screenEndTimeMillis); -// bayesianStruct.posterior.drawVector(bayesianStruct.posterior.getRealTermsAsIndex(0), bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis), bayesPositionWindow); + bayesianStruct.posterior.drawConstrainedVector(startIndex, endIndex, 0, ofGetWidth(), bayesPositionWindow); + + string tmpString = "start "+ofToString(screenStartTimeMillis)+" (index "+ofToString(startIndex)+"), end "+ofToString(screenEndTimeMillis); + ofDrawBitmapString(tmpString, bayesPositionWindow.x+20, bayesPositionWindow.y+20); - bayesianStruct.likelihood.drawVector(bayesianStruct.likelihood.getRealTermsAsIndex(0), bayesianStruct.likelihood.getRealTermsAsIndex(screenWidthMillis), bayesLikelihoodWindow); - bayesianStruct.relativeSpeedPosterior.drawVector(0, bayesianStruct.relativeSpeedPosterior.getRealTermsAsIndex(2), bayesTempoWindow); + bayesianStruct.likelihood.drawConstrainedVector(startIndex, endIndex, 0, ofGetWidth(), bayesLikelihoodWindow); - + + //bayesianStruct.likelihood.drawVector(bayesianStruct.likelihood.getRealTermsAsIndex(0), bayesianStruct.likelihood.getRealTermsAsIndex(screenWidthMillis), bayesLikelihoodWindow); + + bayesianStruct.relativeSpeedPosterior.drawConstrainedVector(0, bayesianStruct.relativeSpeedPosterior.arraySize, 0, ofGetWidth(), bayesTempoWindow); + string tmpStr = "zero is "+ofToString(bayesianStruct.posterior.getRealTermsAsIndex(0)); tmpStr += " offsetis "+ofToString(bayesianStruct.posterior.offset); tmpStr += " screenWidth = "+ofToString(bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis)); @@ -74,8 +98,7 @@ } - -void AudioEventMatcher::newPitchEvent(const double& pitchIn, const double& timeIn){ +void AudioEventMatcher::newPitchEvent(const int& channel, const double& pitchIn, const double& timeIn){ liveInput.addPitchEvent(pitchIn, timeIn); //tmp print stuff @@ -83,25 +106,32 @@ double tmp = bayesianStruct.posterior.getMAPestimate(); printf(" getting it %f and offset %f == %f ms\n", tmp, bayesianStruct.posterior.offset, bayesianStruct.posterior.getIndexInRealTerms(tmp)); - matchNewPitchEvent(0, pitchIn, timeIn); + matchNewPitchEvent(channel, pitchIn, timeIn); } -void AudioEventMatcher::newKickEvent(const double& timeIn){ -// liveInput.addKickEvent(time); +void AudioEventMatcher::newKickEvent(const double& timeIn){ +// liveInput.addKickEvent(timeIn); matchNewOnsetEvent(0, timeIn); } +void AudioEventMatcher::newKickEvent(const int& channel, const double& timeIn){ +// liveInput.addKickEvent(timeIn); + matchNewOnsetEvent(channel, timeIn); +} + void AudioEventMatcher::newSnareEvent(const double& timeIn){ - matchNewOnsetEvent(0, timeIn); + matchNewOnsetEvent(2, timeIn); } //Needs just to set bounds for the matching process, not have TimeIn void AudioEventMatcher::matchNewOnsetEvent(const int& channel, const double& timeIn){ + bayesianStruct.updateBayesianDistributions(timeIn);//moves the posterior up into prior given the time interval and calculates new offsets + //start at beginning but OPTIMISE later - double onsetLikelihoodToNoise = 0.5; + double onsetLikelihoodToNoise = 0.3; double likelihoodWidth = 40; @@ -121,7 +151,7 @@ if (millisTime >= startTime && millisTime <= endTime){ bayesianStruct.likelihood.addGaussianShapeFromRealTime(millisTime, likelihoodWidth, quantity); numberOfMatchesFound++; - printf("Adding Gaussian for onset at time %f offset %f\n", millisTime, bayesianStruct.likelihood.offset); + // printf("Adding Gaussian for onset at time %f offset %f\n", millisTime, bayesianStruct.likelihood.offset); } } @@ -133,6 +163,7 @@ bayesianStruct.likelihood.renormalise(); + } @@ -141,11 +172,11 @@ //start at beginning but OPTIMISE later - updateBayesianDistributions(timeIn); + bayesianStruct.updateBayesianDistributions(timeIn);//moves the posterior up into prior given the time interval and calculates new offsets ///set offsets // bayesianStruct.likelihood.offset = bayesianStruct.prior.offset; - double pitchLikelihoodToNoise = 0.5; + double pitchLikelihoodToNoise = 0.2;//more noise int numberOfMatches = 0; bayesianStruct.likelihood.zero();//set to zero @@ -154,7 +185,7 @@ for (int i = 0;i < recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){ if (checkMatch(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn)) { - quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 40); + quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 20); bayesianStruct.likelihood.addGaussianShapeFromRealTime(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].millisTime, 30, quantity); recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = true; numberOfMatches++; @@ -165,21 +196,16 @@ } } - - bayesianStruct.likelihood.addConstant(numberOfMatches*(1-pitchLikelihoodToNoise)/(pitchLikelihoodToNoise*bayesianStruct.likelihood.length)); - - recordedTracks.recentPitch = pitchIn; - + + if (numberOfMatches > 0){//no point updating unless there is a match + bayesianStruct.likelihood.addConstant(numberOfMatches*(1-pitchLikelihoodToNoise)/(pitchLikelihoodToNoise*bayesianStruct.likelihood.length)); //tmp set likelihood constant and calculate using that - bayesianStruct.likelihood.zero(); - bayesianStruct.likelihood.addConstant(1); + //bayesianStruct.likelihood.zero(); + //bayesianStruct.likelihood.addConstant(1); - bayesianStruct.calculatePosterior(); - - //tmp print stuff - printf("After CALC"); - printPostOffset(); + bayesianStruct.calculatePosterior(); + } } @@ -214,51 +240,3 @@ -void AudioEventMatcher::updateBayesianDistributions(const double& newEventTime){ - //MOVE INTO bayesianStruct?? XX - - //NEED TO CHECK HERE THAT THEY HAVE THE SAME OFFSETS - bayesianStruct.prior.copyFromDynamicVector(bayesianStruct.posterior);//try the otehr way - - //bayesianStruct.copyPriorToPosterior(); - //need to get new MAP position and set the offset of the arrays - //currently bestEstimate is the approx for the new MAP position - int tmpMap = bayesianStruct.posterior.getMAPestimate(); - - double timeDifference = newEventTime - bayesianStruct.lastEventTime; - printf("updating distributions at time %f diff %f offset %f tmpmap est %i\n", newEventTime, timeDifference, bayesianStruct.posterior.offset, tmpMap); - - //addnoise to the tempo distribution - //bayesianStruct.decaySpeedDistribution(timeDifference); - - if (timeDifference > 50){ - bayesianStruct.addGaussianNoiseToSpeedPosterior(timeDifference * 10.0 / 100.); - } - - bayesianStruct.updateBestEstimate(timeDifference); - bayesianStruct.lastBestEstimateUpdateTime = newEventTime;//getTimeNow(timePlayed); - -// printf("set new distrb max 0 or %f\n", bayesianStruct.bestEstimate - (bayesianStruct.prior.scalar*bayesianStruct.prior.arraySize/2)); - - //tmp print stuff - - printf("HALFWAY BEST "); - printPostOffset(); - - bayesianStruct.setNewDistributionOffsets(max(0., bayesianStruct.bestEstimate - (bayesianStruct.prior.scalar*bayesianStruct.prior.arraySize/2))); - bayesianStruct.crossUpdateArrays(bayesianStruct.posterior, bayesianStruct.relativeSpeedPosterior, timeDifference); - - //i.e. using the same offset as prior - bayesianStruct.posterior.offset = bayesianStruct.prior.offset;// - -// float tmpPrior = max(0., bayesianStruct.bestEstimate - (bayesianStruct.prior.scalar*bayesianStruct.prior.arraySize/2));// bayesianStruct.prior.offset = max(0., bayesianStruct.bestEstimate - (bayesianStruct.prior.scalar*bayesianStruct.prior.arraySize/2)); -// printf("Using prior offset of %f not %f\n", tmpPrior, bayesianStruct.prior.offset); - - bayesianStruct.lastEventTime = newEventTime;//bayesianStruct.lastEventTime = ofGetElapsedTimeMillis(); - -} - -void AudioEventMatcher::printPostOffset(){ - double tmp = bayesianStruct.posterior.getMAPestimate(); - printf(" MAP index %i post offset %f == %f ms\n", bayesianStruct.posterior.MAPestimate, bayesianStruct.posterior.offset, bayesianStruct.posterior.getIndexInRealTerms(bayesianStruct.posterior.MAPestimate)); -} \ No newline at end of file
--- a/src/AudioEventMatcher.h Thu Feb 02 12:45:36 2012 +0000 +++ b/src/AudioEventMatcher.h Thu Feb 02 17:52:08 2012 +0000 @@ -28,9 +28,11 @@ void setArraySizes(); void draw(); + void drawBayesianDistributions();; - void newPitchEvent(const double& pitchIn, const double& timeIn); + void newPitchEvent(const int& channel, const double& pitchIn, const double& timeIn); void newKickEvent(const double& timeIn); + void newKickEvent(const int& channel, const double& timeIn); void newSnareEvent(const double& timeIn); void matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn); @@ -50,12 +52,9 @@ bool checkMatch(const double& recordedPitch, const double& livePitch); double getPitchDistance(const double& pitchOne, const double& pitchTwo, const double& scale); - void updateBayesianDistributions(const double& newEventTime); void startPlaying(); bool usingRealTime; - - //tmp debug fn - void printPostOffset(); + }; #endif
--- a/src/RecordedMultitrackAudio.cpp Thu Feb 02 12:45:36 2012 +0000 +++ b/src/RecordedMultitrackAudio.cpp Thu Feb 02 17:52:08 2012 +0000 @@ -13,7 +13,8 @@ void RecordedMultitrackAudio::loadTestAudio(){ - const char *infilename = "../../../data/sound/basicClavScale.wav"; + const char *infilename = "../../../data/sound/LiveDues/neuRoom_liveDues.wav"; + //"../../../data/sound/basicClavScale.wav"; //LoadedAudioHolder lah; // lah.loadAudioFile(infilename); @@ -37,7 +38,7 @@ printf("BEFORE LOADING 1\n"); printInfo(); - infilename = "../../../data/sound/basicClavScale2.wav"; + infilename = "../../../data/sound/LiveDues/bass_upsideLive.wav"; loadedAudioPtr = new LoadedAudioHolder; loadedAudioPtr->loadAudioFile(infilename);
--- a/src/testApp.cpp Thu Feb 02 12:45:36 2012 +0000 +++ b/src/testApp.cpp Thu Feb 02 17:52:08 2012 +0000 @@ -41,8 +41,6 @@ eventMatcher.recordedTracks.updatePosition(); -// audioFilePlayer.updateToPlayPosition(); - checkForOSCmessages(); } @@ -57,17 +55,19 @@ // check for mouse moved message if ( m.getAddress() == "/aubioPitch" ){ - float pitchIn = m.getArgAsFloat(0); - int timeIn = m.getArgAsInt32(1); + int testChannel = m.getArgAsInt32(0); + float pitchIn = m.getArgAsFloat(1); + int timeIn = m.getArgAsInt32(2); printf("\nAUBIO PITCH RECEIVED %f at time %i\n", pitchIn, timeIn); - eventMatcher.newPitchEvent(pitchIn, timeIn); + eventMatcher.newPitchEvent(testChannel, pitchIn, timeIn); } if ( m.getAddress() == "/kick" ){ - // float pitchIn = m.getArgAsFloat(0); - double timeIn = m.getArgAsInt32(0); + // float pitchIn = m.getArgAsFloat(0); + int testChannel = m.getArgAsInt32(0); + double timeIn = m.getArgAsInt32(1); printf("kick RECEIVED at time %f\n", timeIn); - eventMatcher.newKickEvent(timeIn); + eventMatcher.newKickEvent(testChannel, timeIn); } if ( m.getAddress() == "/snare" ){