andrew@0: /* andrew@0: * AudioEventMatcher.cpp andrew@0: * MultipleAudioMathcher andrew@0: * andrew@0: * Created by Andrew on 31/01/2012. andrew@0: * Copyright 2012 QMUL. All rights reserved. andrew@0: * andrew@0: */ andrew@0: andrew@0: #include "AudioEventMatcher.h" andrew@0: andrew@0: andrew@2: const int matchWindowWidth = 6000; andrew@0: andrew@0: AudioEventMatcher::AudioEventMatcher(){ andrew@7: andrew@0: setArraySizes(); andrew@3: andrew@3: usingRealTime = false; andrew@3: bayesianStruct.realTimeMode = &usingRealTime; andrew@7: recentPitch = 0; andrew@8: currentAlignmentPosition = 0; andrew@0: } andrew@0: andrew@7: void AudioEventMatcher::setWindowDimensions(){ andrew@7: double startHeight = recordedTracks.numberOfAudioTracks * recordedTracks.trackScreenHeight; andrew@7: double heightAvailable = 1 - startHeight; andrew@7: heightAvailable /= 3.0; andrew@7: andrew@7: bayesPositionWindow.setToRelativeSize(0, startHeight, 1, heightAvailable); andrew@7: bayesLikelihoodWindow.setToRelativeSize(0, startHeight + 1*heightAvailable, 1, heightAvailable); andrew@7: bayesTempoWindow.setToRelativeSize(0, startHeight + 2*heightAvailable, 1, heightAvailable); andrew@7: andrew@7: andrew@7: } andrew@0: andrew@0: void AudioEventMatcher::setArraySizes(){ andrew@0: bayesianStruct.resetSpeedSize(200); andrew@0: bayesianStruct.setRelativeSpeedScalar(0.01); andrew@0: bayesianStruct.setSpeedPrior(1.0); andrew@0: bayesianStruct.relativeSpeedPrior.getMaximum(); andrew@0: andrew@0: bayesianStruct.resetSize(matchWindowWidth); andrew@0: bayesianStruct.setPositionDistributionScalar(1); andrew@0: andrew@0: } andrew@0: andrew@3: void AudioEventMatcher::startPlaying(){ andrew@3: bayesianStruct.setStartPlaying(); andrew@8: currentAlignmentPosition = 0; andrew@8: startTime = ofGetElapsedTimeMillis(); andrew@3: //bayesianStruct.posterior.printArray(); andrew@3: } andrew@3: andrew@8: void AudioEventMatcher::updateBestAlignmentPosition(){ andrew@8: currentAlignmentPosition = bayesianStruct.posterior.offset + bayesianStruct.posterior.getIndexInRealTerms(bayesianStruct.posterior.MAPestimate); andrew@8: currentAlignmentPosition += (ofGetElapsedTimeMillis() - lastAlignmentTime) * bayesianStruct.relativeSpeedPosterior.getIndexInRealTerms(bayesianStruct.relativeSpeedPosterior.MAPestimate); andrew@8: } andrew@8: andrew@0: void AudioEventMatcher::draw(){ andrew@6: //draw some outlines in blue andrew@3: ofSetColor(20,200,200); andrew@3: bayesPositionWindow.drawOutline(); andrew@3: bayesTempoWindow.drawOutline(); andrew@0: andrew@6: //draw the scrolling audio tracks andrew@1: recordedTracks.drawTracks(); andrew@7: andrew@7: andrew@2: andrew@2: ofSetColor(255); andrew@2: // bayesianStruct.relativeSpeedPrior.drawVector(0, 200, bayesTempoWindow); andrew@2: andrew@6: drawBayesianDistributions(); andrew@8: andrew@6: // bayesianStruct.posterior.drawVector(0, bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis), bayesPositionWindow); andrew@6: andrew@6: //bayesianStruct.posterior.drawVector(bayesianStruct.posterior.getRealTermsAsIndex(0), bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis), bayesPositionWindow); andrew@6: andrew@6: // bayesianStruct.relativeSpeedPosterior.drawVector(0, bayesianStruct.relativeSpeedPosterior.getRealTermsAsIndex(2), bayesTempoWindow); andrew@6: andrew@7: ofDrawBitmapString("pitch "+ofToString(recentPitch, 2)+", Time "+ofToString(recentTime, 0), 20, 20); andrew@6: } andrew@6: andrew@6: void AudioEventMatcher::drawBayesianDistributions(){ andrew@6: andrew@6: andrew@2: double screenWidthMillis = recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.framesToMillis(recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.amplitudeNumber); andrew@2: andrew@6: andrew@6: double screenStartTimeMillis = recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.framesToMillis(recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.drawParams.windowStartFrame); andrew@6: double screenEndTimeMillis = screenStartTimeMillis + screenWidthMillis; andrew@6: int startIndex = bayesianStruct.posterior.getRealTermsAsIndex(screenStartTimeMillis); andrew@6: int endIndex = bayesianStruct.posterior.getRealTermsAsIndex(screenEndTimeMillis); andrew@4: andrew@6: bayesianStruct.posterior.drawConstrainedVector(startIndex, endIndex, 0, ofGetWidth(), bayesPositionWindow); andrew@6: andrew@6: string tmpString = "start "+ofToString(screenStartTimeMillis)+" (index "+ofToString(startIndex)+"), end "+ofToString(screenEndTimeMillis); andrew@6: ofDrawBitmapString(tmpString, bayesPositionWindow.x+20, bayesPositionWindow.y+20); andrew@4: andrew@8: // bayesianStruct.likelihood.drawConstrainedVector(startIndex, endIndex, 0, ofGetWidth(), bayesLikelihoodWindow); andrew@2: andrew@6: bayesianStruct.relativeSpeedPosterior.drawConstrainedVector(0, bayesianStruct.relativeSpeedPosterior.arraySize, 0, ofGetWidth(), bayesTempoWindow); andrew@6: andrew@3: string tmpStr = "zero is "+ofToString(bayesianStruct.posterior.getRealTermsAsIndex(0)); andrew@3: tmpStr += " offsetis "+ofToString(bayesianStruct.posterior.offset); andrew@3: tmpStr += " screenWidth = "+ofToString(bayesianStruct.posterior.getRealTermsAsIndex(screenWidthMillis)); andrew@3: ofDrawBitmapString(tmpStr, 20,140); andrew@3: tmpStr = "best est "+ofToString(bayesianStruct.bestEstimate); andrew@3: ofDrawBitmapString(tmpStr, 20, 180); andrew@3: andrew@8: ofDrawBitmapString("screenwidth "+ofToString(screenWidthMillis), 20, 800); andrew@3: andrew@8: ofSetColor(0,255,0); andrew@8: double currentEstimateIndex = (currentAlignmentPosition - screenStartTimeMillis)*ofGetWidth()/screenWidthMillis; andrew@8: ofLine(currentEstimateIndex, bayesPositionWindow.y, currentEstimateIndex, bayesPositionWindow.y + bayesPositionWindow.height); andrew@7: andrew@7: //draw track by track likelihoods andrew@7: for (int i = 0; i 0){ andrew@1: liveInput.addPitchEvent(pitchIn, timeIn); andrew@4: andrew@4: //tmp print stuff andrew@4: printf("New pitch MAP post estimate now %i, ", bayesianStruct.posterior.MAPestimate); andrew@4: double tmp = bayesianStruct.posterior.getMAPestimate(); andrew@4: printf(" getting it %f and offset %f == %f ms\n", tmp, bayesianStruct.posterior.offset, bayesianStruct.posterior.getIndexInRealTerms(tmp)); andrew@4: andrew@7: matchNewPitchEvent(channel, pitchIn, timeIn);//main pitch matching fn andrew@7: andrew@7: likelihoodVisualisation[1] = bayesianStruct.likelihood; andrew@7: andrew@7: recentPitch = pitchIn;//for drawing andrew@7: recentTime = timeIn; andrew@7: } andrew@8: andrew@8: andrew@2: } andrew@2: andrew@6: void AudioEventMatcher::newKickEvent(const double& timeIn){ andrew@6: // liveInput.addKickEvent(timeIn); andrew@2: matchNewOnsetEvent(0, timeIn); andrew@7: likelihoodVisualisation[0] = bayesianStruct.likelihood; andrew@2: } andrew@2: andrew@6: void AudioEventMatcher::newKickEvent(const int& channel, const double& timeIn){ andrew@6: // liveInput.addKickEvent(timeIn); andrew@6: matchNewOnsetEvent(channel, timeIn); andrew@7: likelihoodVisualisation[0] = bayesianStruct.likelihood; andrew@6: } andrew@6: andrew@2: andrew@2: void AudioEventMatcher::newSnareEvent(const double& timeIn){ andrew@6: matchNewOnsetEvent(2, timeIn); andrew@7: likelihoodVisualisation[2] = bayesianStruct.likelihood; andrew@7: } andrew@7: andrew@7: andrew@7: void AudioEventMatcher::newSnareEvent(const int& channel, const double& timeIn){ andrew@7: matchNewOnsetEvent(channel, timeIn); andrew@7: likelihoodVisualisation[2] = bayesianStruct.likelihood; andrew@2: } andrew@2: andrew@2: //Needs just to set bounds for the matching process, not have TimeIn andrew@2: void AudioEventMatcher::matchNewOnsetEvent(const int& channel, const double& timeIn){ andrew@3: andrew@3: andrew@6: bayesianStruct.updateBayesianDistributions(timeIn);//moves the posterior up into prior given the time interval and calculates new offsets andrew@6: andrew@2: //start at beginning but OPTIMISE later andrew@8: double onsetLikelihoodToNoise = 0.5; andrew@2: andrew@2: double likelihoodWidth = 40; andrew@2: andrew@2: bayesianStruct.likelihood.offset = bayesianStruct.prior.offset; andrew@2: bayesianStruct.likelihood.zero();//set to zero andrew@2: andrew@2: double quantity = 1;//likelihoodToNoiseRatio / numberOfMatches; andrew@2: int numberOfMatchesFound = 0; andrew@2: andrew@2: andrew@2: double startTime = bayesianStruct.likelihood.offset; andrew@2: double endTime = bayesianStruct.likelihood.offset + matchWindowWidth; andrew@2: andrew@2: if (channel <= recordedTracks.numberOfAudioTracks){ andrew@2: for (int i = 0;i < recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){ andrew@2: double millisTime = recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].millisTime; andrew@2: if (millisTime >= startTime && millisTime <= endTime){ andrew@2: bayesianStruct.likelihood.addGaussianShapeFromRealTime(millisTime, likelihoodWidth, quantity); andrew@2: numberOfMatchesFound++; andrew@6: // printf("Adding Gaussian for onset at time %f offset %f\n", millisTime, bayesianStruct.likelihood.offset); andrew@2: andrew@2: } andrew@2: } andrew@2: } andrew@2: andrew@3: // bayesianStruct.likelihood.addConstant((1-likelihoodToNoiseRatio)/bayesianStruct.likelihood.length); andrew@3: bayesianStruct.likelihood.addConstant(numberOfMatchesFound*(1-onsetLikelihoodToNoise)/(onsetLikelihoodToNoise*bayesianStruct.likelihood.length)); andrew@2: bayesianStruct.likelihood.renormalise(); andrew@2: andrew@8: bayesianStruct.calculatePosterior(); andrew@8: andrew@8: lastAlignmentTime = ofGetElapsedTimeMillis(); andrew@8: recentEventTime[channel] = ofGetElapsedTimeMillis() - startTime; andrew@6: andrew@3: } andrew@3: andrew@3: andrew@3: andrew@3: void AudioEventMatcher::matchNewPitchEvent(const int& channel, const double& pitchIn, const double& timeIn){ andrew@3: //start at beginning but OPTIMISE later andrew@2: andrew@2: andrew@6: bayesianStruct.updateBayesianDistributions(timeIn);//moves the posterior up into prior given the time interval and calculates new offsets andrew@8: andrew@7: //set the lielihoods by matching the pitched note andrew@7: andrew@8: double pitchLikelihoodToNoise = 0.5;//more noise andrew@3: int numberOfMatches = 0; andrew@3: bayesianStruct.likelihood.zero();//set to zero andrew@3: andrew@3: double quantity = 0; andrew@3: if (channel <= recordedTracks.numberOfAudioTracks){ andrew@3: for (int i = 0;i < recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){ andrew@3: andrew@3: if (checkMatch(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn)) { andrew@7: quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 10); andrew@3: bayesianStruct.likelihood.addGaussianShapeFromRealTime(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].millisTime, 30, quantity); andrew@3: recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = true; andrew@3: numberOfMatches++; andrew@3: } andrew@3: else{ andrew@3: recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = false; andrew@3: } andrew@3: andrew@3: } andrew@3: } andrew@6: andrew@8: recentPrior = bayesianStruct.prior; andrew@8: andrew@8: andrew@6: if (numberOfMatches > 0){//no point updating unless there is a match andrew@7: andrew@6: bayesianStruct.likelihood.addConstant(numberOfMatches*(1-pitchLikelihoodToNoise)/(pitchLikelihoodToNoise*bayesianStruct.likelihood.length)); andrew@4: andrew@4: //tmp set likelihood constant and calculate using that andrew@6: //bayesianStruct.likelihood.zero(); andrew@6: //bayesianStruct.likelihood.addConstant(1); andrew@7: andrew@6: bayesianStruct.calculatePosterior(); andrew@6: } andrew@4: andrew@8: lastAlignmentTime = ofGetElapsedTimeMillis(); andrew@8: recentEventTime[channel] = ofGetElapsedTimeMillis() - startTime; andrew@1: } andrew@1: andrew@3: double AudioEventMatcher::getPitchDistance(const double& pitchOne, const double& pitchTwo, const double& scale){ andrew@3: andrew@3: double distance = abs(pitchOne - pitchTwo); andrew@3: if (distance < scale) andrew@3: distance = 1 - (distance/scale); andrew@3: else andrew@3: distance = 0; andrew@3: andrew@3: // printf("[pitch distance %f vs %f = %f\n", pitchOne, pitchTwo, distance); andrew@3: return distance; andrew@3: andrew@3: } andrew@3: andrew@3: andrew@3: bool AudioEventMatcher::checkMatch(const double& recordedPitch, const double& livePitch){ andrew@3: if (abs(recordedPitch - livePitch) < 40) andrew@3: return true; andrew@3: else andrew@3: return false; andrew@3: } andrew@3: andrew@3: andrew@1: andrew@1: void AudioEventMatcher::windowResized(const int& w, const int& h){ andrew@1: recordedTracks.windowResized(w,h); andrew@3: bayesTempoWindow.resized(w,h); andrew@3: bayesPositionWindow.resized(w,h); andrew@3: } andrew@3: andrew@3: andrew@3: