Mercurial > hg > multitrack-audio-matcher

changeset 16:680ba08e9925

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Auto synchroniser added using the elastique~ object in max
author Andrew N Robertson <andrew.robertson@eecs.qmul.ac.uk>
date Mon, 06 Feb 2012 12:18:40 +0000
parents 780def3a1f36
children c96b18dd0f48
files src/AccompanimentSynchroniser.cpp src/AccompanimentSynchroniser.h src/AudioEventMatcher.cpp src/AudioEventMatcher.h src/RecordedMultitrackAudio.cpp src/testApp.cpp
diffstat 6 files changed, 154 insertions(+), 13 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/AccompanimentSynchroniser.cpp	Mon Feb 06 12:18:40 2012 +0000
@@ -0,0 +1,53 @@
+/*
+ *  AccompanimentSynchroniser.cpp
+ *  MultipleAudioMathcher
+ *
+ *  Created by Andrew on 06/02/2012.
+ *  Copyright 2012 QMUL. All rights reserved.
+ *
+ */
+
+#include "AccompanimentSynchroniser.h"
+
+AccompanimentSynchroniser::AccompanimentSynchroniser(){
+	sender.setup(HOST,  SENDER_PORT);
+}
+
+void AccompanimentSynchroniser::setPlayingRatio(const double& ratio, const double& timePlayed){
+	playingPositionRatio = ratio;
+	playingPositionSamples = ratio * fileLengthSamples;
+	playingPositionMillis = playingPositionSamples * 1000.0/44100.0;
+	playingPositionTimeSent = timePlayed;
+}
+
+
+void AccompanimentSynchroniser::updateRecordedPosition(const double& currentAlignmentPosition){
+	recordedPositionMillis = currentAlignmentPosition;
+}
+
+
+void AccompanimentSynchroniser::updateOutputSpeed(){
+	//we want the playing position to more closely align with the recordedPosition
+	
+	double difference = recordedPositionMillis - playingPositionMillis;
+	
+	//suppose we project that we will align in two seconds time
+	double projectedAlignmentTime = 2000;
+	
+	//then our current speed projects forward
+	double predictedPlayingTimeIncrease = projectedAlignmentTime*speed;
+	
+	difference += (projectedAlignmentTime - predictedPlayingTimeIncrease);
+	
+	speed += difference/projectedAlignmentTime;
+	
+	sendSpeed(speed);
+
+}
+
+void AccompanimentSynchroniser::sendSpeed(double const& val){
+	ofxOscMessage m;
+	m.setAddress( "/setSpeed" );
+	m.addFloatArg( val );
+	sender.sendMessage( m );
+}
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/AccompanimentSynchroniser.h	Mon Feb 06 12:18:40 2012 +0000
@@ -0,0 +1,51 @@
+/*
+ *  AccompanimentSynchroniser.h
+ *  MultipleAudioMathcher
+ *
+ *  Created by Andrew on 06/02/2012.
+ *  Copyright 2012 QMUL. All rights reserved.
+ *
+ */
+
+
+#ifndef ACCOMPANIMENT_SYNCHRONISER_H
+#define ACCOMPANIMENT_SYNCHRONISER_H
+
+#include "ofxOsc.h"
+#define HOST "localhost"
+#define SENDER_PORT 10875
+
+#include "ofMain.h"
+
+
+class AccompanimentSynchroniser{
+	
+public:
+	AccompanimentSynchroniser();
+	
+	void setPlayingRatio(const double& ratio, const double& timePlayed);
+	void setRecordedPosition();
+	void updateRecordedPosition(const double& currentAlignmentPosition);
+	
+	void updateOutputSpeed();
+	
+	double fileLengthSamples;
+	
+
+	
+	double playingPositionRatio;
+	double playingPositionSamples;
+	double playingPositionMillis;
+	double playingPositionTimeSent;
+	
+private:
+	
+	double recordedPositionMillis;
+	
+	double speed;
+	
+	ofxOscSender sender;
+	void sendSpeed(double const& val);
+
+};
+#endif
--- a/src/AudioEventMatcher.cpp	Sun Feb 05 22:08:50 2012 +0000
+++ b/src/AudioEventMatcher.cpp	Mon Feb 06 12:18:40 2012 +0000
@@ -15,8 +15,9 @@
 AudioEventMatcher::AudioEventMatcher(){
 
 	
-	pitchLikelihoodToNoise = 0.4;//more noise
-	onsetLikelihoodToNoise = 0.3;
+	pitchLikelihoodToNoise = 0.9;//more noise
+	
+	onsetLikelihoodToNoise = 0.4;
 	onsetLikelihoodWidth = 20;//in ms
 	
 	setArraySizes();
@@ -56,6 +57,12 @@
 	
 }
 
+void AudioEventMatcher::loadAudioFiles(){
+	recordedTracks.loadTestAudio();
+	synchroniser.fileLengthSamples = recordedTracks.loadedAudioFiles[0].fileLoader.totalNumberOfSamples;
+	printf("synchroniser has %f samples\n", synchroniser.fileLengthSamples);
+}
+
 void AudioEventMatcher::startPlaying(){
 	bayesianStruct.setStartPlaying();
 	currentAlignmentPosition = 0;
@@ -92,6 +99,11 @@
 	currentAlignmentPosition = bayesianStruct.posterior.getIndexInRealTerms(bayesianStruct.posterior.MAPestimate);
 	currentAlignmentPosition += (newTime - lastAlignmentTime) * bayesianStruct.relativeSpeedPosterior.getIndexInRealTerms(bayesianStruct.relativeSpeedPosterior.MAPestimate);
 
+	
+	synchroniser.updateRecordedPosition(currentAlignmentPosition);
+	
+	synchroniser.updateOutputSpeed();
+	
 	bayesianStruct.projectDistribution(newTime, currentAlignmentPosition, projectedPrior);//prior gets updated to where we are now
 	
 //	printf("ALIGN pos %f time diff %f (now %f , last %f)speed %f :: ALIGN BEST %f\n", tmp, timetmp, (double)ofGetElapsedTimeMillis(), lastAlignmentTime, speedtmp, currentAlignmentPosition);
@@ -118,6 +130,11 @@
 	
 	ofDrawBitmapString("pitch "+ofToString(recentPitch, 2)+", Time "+ofToString(recentTime, 0), 20, 20);
 	
+	string alignString = " align "+ofToString(currentAlignmentPosition, 2);
+	alignString += " playing "+ofToString(synchroniser.playingPositionRatio, 5);
+	alignString += "  "+ofToString(synchroniser.playingPositionMillis)+" ms";
+	ofDrawBitmapString(alignString, 20, 50);
+	
 	ofDrawBitmapString("pos "+ofToString(recordedTracks.loadedAudioFiles[0].fileLoader.onsetDetect.playPosition), 200,600);
 }
 	
@@ -167,6 +184,13 @@
 	double currentEstimateIndex = (currentAlignmentPosition - screenStartTimeMillis)*ofGetWidth()/screenWidthMillis;
 	ofLine(currentEstimateIndex, bayesPositionWindow.y, currentEstimateIndex, bayesPositionWindow.y + bayesPositionWindow.height);
 	
+
+	ofSetColor(0,255,255);//synchroniser position
+	currentEstimateIndex = (synchroniser.playingPositionMillis - screenStartTimeMillis)*ofGetWidth()/screenWidthMillis;
+	ofLine(currentEstimateIndex, bayesLikelihoodWindow.y, currentEstimateIndex, bayesLikelihoodWindow.y + bayesPositionWindow.height);
+	
+	
+	
 	//draw track by track likelihoods
 	for (int i = 0; i <recordedTracks.numberOfAudioTracks;i++){
 		ofSetColor(200,255,50);//channel likelihoods in yellow
@@ -302,7 +326,7 @@
 		for (int i = 0;i < recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets.size();i++){
 			
 			if (checkMatch(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn)) {
-				quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 10);
+				quantity = getPitchDistance(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].aubioPitch, pitchIn, 16);
 				bayesianStruct.likelihood.addGaussianShapeFromRealTime(recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].millisTime, 30, quantity);
 				recordedTracks.loadedAudioFiles[channel].fileLoader.onsetDetect.chromaOnsets[i].matched = true;
 				numberOfMatches++;
@@ -337,9 +361,12 @@
 
 double AudioEventMatcher::getPitchDistance(const double& pitchOne, const double& pitchTwo, const double& scale){
 	
+	double scaleFactor = scale ;//* pitchOne / 110.0;
+	
+	
 	double distance = abs(pitchOne - pitchTwo);
-	if (distance < scale)
-		distance = 1 - (distance/scale);
+	if (distance < scaleFactor)
+		distance = 1 - (distance/scaleFactor);
 	else
 		distance = 0;
 	
@@ -350,7 +377,7 @@
 
 
 bool AudioEventMatcher::checkMatch(const double& recordedPitch, const double& livePitch){
-	if (abs(recordedPitch - livePitch) < 40)
+	if (abs(recordedPitch - livePitch) < 16)
 		return true;
 	else
 		return false;
--- a/src/AudioEventMatcher.h	Sun Feb 05 22:08:50 2012 +0000
+++ b/src/AudioEventMatcher.h	Mon Feb 06 12:18:40 2012 +0000
@@ -20,6 +20,7 @@
 #include "BayesianArrayStructure.h"
 #include "RecordedMultitrackAudio.h"
 #include "DynamicVector.h"
+#include "AccompanimentSynchroniser.h"
 
 class AudioEventMatcher{
 	
@@ -49,6 +50,8 @@
 	LiveAudioInput liveInput;//hold the new events that come in
 	RecordedMultitrackAudio recordedTracks;
 	
+	void loadAudioFiles();
+	
 	void windowResized(const int& w, const int& h);
 	
 	ofxWindowRegion bayesTempoWindow;
@@ -81,6 +84,8 @@
 	bool followingLiveInput;
 	void setScreenDisplayTimes();
 	
+	AccompanimentSynchroniser synchroniser;
+	
 	//params
 	double onsetLikelihoodWidth;
 	double onsetLikelihoodToNoise;
--- a/src/RecordedMultitrackAudio.cpp	Sun Feb 05 22:08:50 2012 +0000
+++ b/src/RecordedMultitrackAudio.cpp	Mon Feb 06 12:18:40 2012 +0000
@@ -21,14 +21,14 @@
 //	const char	*bassfilename = "../../../data/sound/DiamondMatch1/bass_bip.wav";	
 //	const char	*snarefilename = "../../../data/sound/DiamondMatch1/snare_bip.wav";	
 	
-//	const char	*kickfilename = "../../../data/sound/LiveIdiot2/02kick_bip.wav";	
-//	const char	*bassfilename = "../../../data/sound/LiveIdiot2/01bass_bip.wav";	
-//	const char	*snarefilename = "../../../data/sound/LiveIdiot2/03snare_bip.wav";	
+	const char	*kickfilename = "../../../data/sound/LiveIdiot2/02kick_bip.wav";	
+	const char	*bassfilename = "../../../data/sound/LiveIdiot2/01bass_bip.wav";	
+	const char	*snarefilename = "../../../data/sound/LiveIdiot2/03snare_bip.wav";	
 	
-	const char	*kickfilename = "../../../data/sound/UistLive1/02kick_bip.wav";	
+/*	const char	*kickfilename = "../../../data/sound/UistLive1/02kick_bip.wav";	
 	const char	*bassfilename = "../../../data/sound/UistLive1/01bass_bip.wav";	
 	const char	*snarefilename = "../../../data/sound/UistLive1/03snare_bip.wav";		
-	
+*/	
 	//"../../../data/sound/basicClavScale.wav";	
 	
 	//LoadedAudioHolder lah;
--- a/src/testApp.cpp	Sun Feb 05 22:08:50 2012 +0000
+++ b/src/testApp.cpp	Mon Feb 06 12:18:40 2012 +0000
@@ -28,8 +28,7 @@
 	ofSetFrameRate(30);
 
 	
-	
-	eventMatcher.recordedTracks.loadTestAudio();
+	eventMatcher.loadAudioFiles();
 	
 	eventMatcher.setWindowDimensions();
 	//audioFilePlayer.loadAudioFile(infilename);
@@ -92,6 +91,12 @@
 			eventMatcher.stopPlaying();
 		}
 		
+		if ( m.getAddress() == "/accompanimentRatio" ){
+			double time = m.getArgAsFloat(0);
+			double ratio = m.getArgAsFloat(1);
+			eventMatcher.synchroniser.setPlayingRatio(ratio, time);
+		}
+		
 	}
 }