# HG changeset patch # User Chris Cannam # Date 1234270363 0 # Node ID c8908cdc8c32f14c837dc116a482c88b8bfa8973 # Parent 833ca65b08203373c35a57acf4220fde11cf8dcb * First cut at Matthew's downbeat estimator -- untested so far diff -r 833ca65b0820 -r c8908cdc8c32 dsp/onsets/DetectionFunction.cpp --- a/dsp/onsets/DetectionFunction.cpp Mon Feb 09 16:05:32 2009 +0000 +++ b/dsp/onsets/DetectionFunction.cpp Tue Feb 10 12:52:43 2009 +0000 @@ -38,7 +38,6 @@ m_halfLength = m_dataLength/2; m_DFType = Config.DFType; - m_stepSecs = Config.stepSecs; m_stepSize = Config.stepSize; m_whiten = Config.adaptiveWhitening; diff -r 833ca65b0820 -r c8908cdc8c32 dsp/onsets/DetectionFunction.h --- a/dsp/onsets/DetectionFunction.h Mon Feb 09 16:05:32 2009 +0000 +++ b/dsp/onsets/DetectionFunction.h Tue Feb 10 12:52:43 2009 +0000 @@ -23,7 +23,6 @@ #define DF_BROADBAND (5) struct DFConfig{ - double stepSecs; // DF step in seconds unsigned int stepSize; // DF step in samples unsigned int frameLength; // DF analysis window - usually 2*step int DFType; // type of detection function ( see defines ) @@ -59,7 +58,6 @@ int m_DFType; unsigned int m_dataLength; unsigned int m_halfLength; - double m_stepSecs; unsigned int m_stepSize; double m_dbRise; bool m_whiten; diff -r 833ca65b0820 -r c8908cdc8c32 dsp/rateconversion/Decimator.h --- a/dsp/rateconversion/Decimator.h Mon Feb 09 16:05:32 2009 +0000 +++ b/dsp/rateconversion/Decimator.h Tue Feb 10 12:52:43 2009 +0000 @@ -16,6 +16,16 @@ void process( const double* src, double* dst ); void doAntiAlias( const double* src, double* dst, unsigned int length ); + /** + * Construct a Decimator to operate on input blocks of length + * inLength, with decimation factor decFactor. inLength should be + * a multiple of decFactor. Output blocks will be of length + * inLength / decFactor. + * + * decFactor must be a power of two. The highest supported factor + * is obtained through getHighestSupportedFactor(); for higher + * factors, you will need to chain more than one decimator. + */ Decimator( unsigned int inLength, unsigned int decFactor ); virtual ~Decimator(); diff -r 833ca65b0820 -r c8908cdc8c32 dsp/tempotracking/DownBeat.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dsp/tempotracking/DownBeat.cpp Tue Feb 10 12:52:43 2009 +0000 @@ -0,0 +1,241 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + QM DSP Library + + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2008-2009 Matthew Davies and QMUL. + All rights reserved. +*/ + +#include "DownBeat.h" + +#include "maths/MathAliases.h" +#include "maths/MathUtilities.h" +#include "dsp/transforms/FFT.h" + +#include +#include + +DownBeat::DownBeat(float originalSampleRate, + size_t decimationFactor, + size_t dfIncrement) : + m_rate(originalSampleRate), + m_factor(decimationFactor), + m_increment(dfIncrement), + m_decimator1(0), + m_decimator2(0), + m_buffer(0), + m_bufsiz(0), + m_buffill(0), + m_beatframesize(0), + m_beatframe(0) +{ + // beat frame size is next power of two up from 1.3 seconds at the + // downsampled rate (happens to produce 4096 for 44100 or 48000 at + // 16x decimation, which is our expected normal situation) + int bfs = int((m_rate / decimationFactor) * 1.3); + m_beatframesize = 1; + while (bfs) { bfs >>= 1; m_beatframesize <<= 1; } + std::cerr << "rate = " << m_rate << ", bfs = " << m_beatframesize << std::endl; + m_beatframe = new double[m_beatframesize]; + m_fftRealOut = new double[m_beatframesize]; + m_fftImagOut = new double[m_beatframesize]; +} + +DownBeat::~DownBeat() +{ + delete m_decimator1; + delete m_decimator2; + if (m_buffer) free(m_buffer); + delete[] m_decbuf; + delete[] m_beatframe; + delete[] m_fftRealOut; + delete[] m_fftImagOut; +} + +void +DownBeat::makeDecimators() +{ + if (m_factor < 2) return; + int highest = Decimator::getHighestSupportedFactor(); + if (m_factor <= highest) { + m_decimator1 = new Decimator(m_increment, m_factor); + return; + } + m_decimator1 = new Decimator(m_increment, highest); + m_decimator2 = new Decimator(m_increment / highest, m_factor / highest); + m_decbuf = new double[m_factor / highest]; +} + +void +DownBeat::pushAudioBlock(const double *audio) +{ + if (m_buffill + (m_increment / m_factor) > m_bufsiz) { + if (m_bufsiz == 0) m_bufsiz = m_increment * 16; + else m_bufsiz = m_bufsiz * 2; + if (!m_buffer) { + m_buffer = (double *)malloc(m_bufsiz * sizeof(double)); + } else { + std::cerr << "DownBeat::pushAudioBlock: realloc m_buffer to " << m_bufsiz << std::endl; + m_buffer = (double *)realloc(m_buffer, m_bufsiz * sizeof(double)); + } + } + if (!m_decimator1) makeDecimators(); + if (m_decimator2) { + m_decimator1->process(audio, m_decbuf); + m_decimator2->process(m_decbuf, m_buffer + m_buffill); + } else { + m_decimator1->process(audio, m_buffer + m_buffill); + } + m_buffill += m_increment / m_factor; +} + +const double * +DownBeat::getBufferedAudio(size_t &length) const +{ + length = m_buffill; + return m_buffer; +} + +void +DownBeat::findDownBeats(const double *audio, + size_t audioLength, + const d_vec_t &beats, + i_vec_t &downbeats) +{ + // FIND DOWNBEATS BY PARTITIONING THE INPUT AUDIO FILE INTO BEAT SEGMENTS + // WHERE THE AUDIO FRAMES ARE DOWNSAMPLED BY A FACTOR OF 16 (fs ~= 2700Hz) + // THEN TAKING THE JENSEN-SHANNON DIVERGENCE BETWEEN BEAT SYNCHRONOUS SPECTRAL FRAMES + + // IMPLEMENTATION (MOSTLY) FOLLOWS: + // DAVIES AND PLUMBLEY "A SPECTRAL DIFFERENCE APPROACH TO EXTRACTING DOWNBEATS IN MUSICAL AUDIO" + // EUSIPCO 2006, FLORENCE, ITALY + + d_vec_t newspec(m_beatframesize / 2); // magnitude spectrum of current beat + d_vec_t oldspec(m_beatframesize / 2); // magnitude spectrum of previous beat + d_vec_t specdiff; + + if (audioLength == 0) return; + + for (size_t i = 0; i + 1 < beats.size(); ++i) { + + // Copy the extents of the current beat from downsampled array + // into beat frame buffer + + size_t beatstart = (beats[i] * m_increment) / m_factor; + size_t beatend = (beats[i] * m_increment) / m_factor; + if (beatend >= audioLength) beatend = audioLength - 1; + if (beatend < beatstart) beatend = beatstart; + size_t beatlen = beatend - beatstart; + + // Also apply a Hanning window to the beat frame buffer, sized + // to the beat extents rather than the frame size. (Because + // the size varies, it's easier to do this by hand than use + // our Window abstraction.) + + for (size_t j = 0; j < beatlen; ++j) { + double mul = 0.5 * (1.0 - cos(TWO_PI * (double(j) / double(beatlen)))); + m_beatframe[j] = audio[beatstart + j] * mul; + } + + for (size_t j = beatlen; j < m_beatframesize; ++j) { + m_beatframe[j] = 0.0; + } + + // Now FFT beat frame + + FFT::process(m_beatframesize, false, + m_beatframe, 0, m_fftRealOut, m_fftImagOut); + + // Calculate magnitudes + + for (size_t j = 0; j < m_beatframesize/2; ++j) { + newspec[j] = sqrt(m_fftRealOut[j] * m_fftRealOut[j] + + m_fftImagOut[j] * m_fftImagOut[j]); + } + + // Preserve peaks by applying adaptive threshold + + MathUtilities::adaptiveThreshold(newspec); + + // Calculate JS divergence between new and old spectral frames + + specdiff.push_back(measureSpecDiff(oldspec, newspec)); + + // Copy newspec across to old + + for (size_t j = 0; j < m_beatframesize/2; ++j) { + oldspec[j] = newspec[j]; + } + } + + // We now have all spectral difference measures in specdiff + + uint timesig = 4; // SHOULD REPLACE THIS WITH A FIND_METER FUNCTION - OR USER PARAMETER + d_vec_t dbcand(timesig); // downbeat candidates + + // look for beat transition which leads to greatest spectral change + for (int beat = 0; beat < timesig; ++beat) { + for (int example = beat; example < specdiff.size(); ++example) { + dbcand[beat] += (specdiff[example]) / timesig; + } + } + + // first downbeat is beat at index of maximum value of dbcand + int dbind = MathUtilities::getMax(dbcand); + + // remaining downbeats are at timesig intervals from the first + for (int i = dbind; i < beats.size(); i += timesig) { + downbeats.push_back(i); + } +} + +double +DownBeat::measureSpecDiff(d_vec_t oldspec, d_vec_t newspec) +{ + // JENSEN-SHANNON DIVERGENCE BETWEEN SPECTRAL FRAMES + + uint SPECSIZE = 512; // ONLY LOOK AT FIRST 512 SAMPLES OF SPECTRUM. + if (SPECSIZE > oldspec.size()/4) { + SPECSIZE = oldspec.size()/4; + } + double SD = 0.; + double sd1 = 0.; + + double sumnew = 0.; + double sumold = 0.; + + for (uint i = 0;i < SPECSIZE;i++) + { + newspec[i] +=EPS; + oldspec[i] +=EPS; + + sumnew+=newspec[i]; + sumold+=oldspec[i]; + } + + for (uint i = 0;i < SPECSIZE;i++) + { + newspec[i] /= (sumnew); + oldspec[i] /= (sumold); + + // IF ANY SPECTRAL VALUES ARE 0 (SHOULDN'T BE ANY!) SET THEM TO 1 + if (newspec[i] == 0) + { + newspec[i] = 1.; + } + + if (oldspec[i] == 0) + { + oldspec[i] = 1.; + } + + // JENSEN-SHANNON CALCULATION + sd1 = 0.5*oldspec[i] + 0.5*newspec[i]; + SD = SD + (-sd1*log(sd1)) + (0.5*(oldspec[i]*log(oldspec[i]))) + (0.5*(newspec[i]*log(newspec[i]))); + } + + return SD; +} + diff -r 833ca65b0820 -r c8908cdc8c32 dsp/tempotracking/DownBeat.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dsp/tempotracking/DownBeat.h Tue Feb 10 12:52:43 2009 +0000 @@ -0,0 +1,107 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + QM DSP Library + + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2008-2009 Matthew Davies and QMUL. + All rights reserved. +*/ + +#ifndef DOWNBEAT_H +#define DOWNBEAT_H + +#include + +#include "dsp/rateconversion/Decimator.h" + +using std::vector; + +/** + * This class takes an input audio signal and a sequence of beat + * locations (calculated e.g. by TempoTrackV2) and estimates which of + * the beat locations are downbeats (first beat of the bar). + * + * The input audio signal is expected to have been downsampled to a + * very low sampling rate (e.g. 2700Hz). A utility function for + * downsampling and buffering incoming block-by-block audio is + * provided. + */ +class DownBeat +{ +public: + /** + * Construct a downbeat locator that will operate on audio at the + * downsampled by the given decimation factor from the given + * original sample rate, plus beats extracted from the same audio + * at the given original sample rate with the given frame + * increment. + * + * decimationFactor must be a power of two no greater than 64, and + * dfIncrement must be a multiple of decimationFactor. + */ + DownBeat(float originalSampleRate, + size_t decimationFactor, + size_t dfIncrement); + ~DownBeat(); + + /** + * Estimate which beats are down-beats. + * + * audio contains the input audio stream after downsampling, and + * audioLength contains the number of samples in this downsampled + * stream. + * + * beats contains a series of beat positions expressed in + * multiples of the df increment at the audio's original sample + * rate, as described to the constructor. + * + * The returned downbeat array contains a series of indices to the + * beats array. + */ + void findDownBeats(const double *audio, // downsampled + size_t audioLength, // after downsampling + const vector &beats, + vector &downbeats); + + /** + * For your downsampling convenience: call this function + * repeatedly with input audio blocks containing dfIncrement + * samples at the original sample rate, to decimate them to the + * downsampled rate and buffer them within the DownBeat class. + * + * Call getBufferedAudio() to retrieve the results after all + * blocks have been processed. + */ + void pushAudioBlock(const double *audio); + + /** + * Retrieve the accumulated audio produced by pushAudioBlock calls. + */ + const double *getBufferedAudio(size_t &length) const; + +private: + typedef vector i_vec_t; + typedef vector > i_mat_t; + typedef vector d_vec_t; + typedef vector > d_mat_t; + + void makeDecimators(); + double measureSpecDiff(d_vec_t oldspec, d_vec_t newspec); + + float m_rate; + size_t m_factor; + size_t m_increment; + Decimator *m_decimator1; + Decimator *m_decimator2; + double *m_buffer; + double *m_decbuf; + size_t m_bufsiz; + size_t m_buffill; + size_t m_beatframesize; + double *m_beatframe; + double *m_fftRealOut; + double *m_fftImagOut; +}; + +#endif diff -r 833ca65b0820 -r c8908cdc8c32 dsp/tempotracking/TempoTrackV2.cpp --- a/dsp/tempotracking/TempoTrackV2.cpp Mon Feb 09 16:05:32 2009 +0000 +++ b/dsp/tempotracking/TempoTrackV2.cpp Tue Feb 10 12:52:43 2009 +0000 @@ -14,69 +14,15 @@ #include #include +#include "maths/MathUtilities.h" -//#define FRAMESIZE 512 -//#define BIGFRAMESIZE 1024 -#define TWOPI 6.283185307179586232 #define EPS 0.0000008 // just some arbitrary small number -TempoTrackV2::TempoTrackV2() { } +TempoTrackV2::TempoTrackV2(float rate, size_t increment) : + m_rate(rate), m_increment(increment) { } TempoTrackV2::~TempoTrackV2() { } void -TempoTrackV2::adapt_thresh(d_vec_t &df) -{ - d_vec_t smoothed(df.size()); - - int p_post = 7; - int p_pre = 8; - - int t = std::min(static_cast(df.size()),p_post); // what is smaller, p_post of df size. This is to avoid accessing outside of arrays - - // find threshold for first 't' samples, where a full average cannot be computed yet - for (int i = 0;i <= t;i++) - { - int k = std::min((i+p_pre),static_cast(df.size())); - smoothed[i] = mean_array(df,1,k); - } - // find threshold for bulk of samples across a moving average from [i-p_pre,i+p_post] - for (uint i = t+1;i < df.size()-p_post;i++) - { - smoothed[i] = mean_array(df,i-p_pre,i+p_post); - } - // for last few samples calculate threshold, again, not enough samples to do as above - for (uint i = df.size()-p_post;i < df.size();i++) - { - int k = std::max((static_cast (i) -p_post),1); - smoothed[i] = mean_array(df,k,df.size()); - } - - // subtract the threshold from the detection function and check that it is not less than 0 - for (uint i = 0;i < df.size();i++) - { - df[i] -= smoothed[i]; - if (df[i] < 0) - { - df[i] = 0; - } - } -} - -double -TempoTrackV2::mean_array(const d_vec_t &dfin,int start,int end) -{ - double sum = 0.; - - // find sum - for (int i = start;i < end;i++) - { - sum += dfin[i]; - } - - return static_cast (sum / (end - start + 1) ); // average and return -} - -void TempoTrackV2::filter_df(d_vec_t &df) { d_vec_t a(3); @@ -205,7 +151,7 @@ d_vec_t dfframe(dfframe_in); - adapt_thresh(dfframe); + MathUtilities::adaptiveThreshold(dfframe); d_vec_t acf(dfframe.size()); @@ -238,7 +184,7 @@ } // apply adaptive threshold to rcf - adapt_thresh(rcf); + MathUtilities::adaptiveThreshold(rcf); double rcfsum =0.; for (uint i=0; i &df, vector &beatPeriod, vector &tempi); + // Returned beat positions are given in df increment units void calculateBeats(const vector &df, const vector &beatPeriod, vector &beats); @@ -36,6 +51,9 @@ typedef vector d_vec_t; typedef vector > d_mat_t; + float m_rate; + size_t m_increment; + void adapt_thresh(d_vec_t &df); double mean_array(const d_vec_t &dfin, int start, int end); void filter_df(d_vec_t &df); diff -r 833ca65b0820 -r c8908cdc8c32 maths/MathAliases.h --- a/maths/MathAliases.h Mon Feb 09 16:05:32 2009 +0000 +++ b/maths/MathAliases.h Tue Feb 10 12:52:43 2009 +0000 @@ -22,7 +22,7 @@ #define PI (3.14159265358979232846) #endif -#define TWO_PI (*2.PI) +#define TWO_PI (2. * PI) #define EPS 2.2204e-016 diff -r 833ca65b0820 -r c8908cdc8c32 maths/MathUtilities.cpp --- a/maths/MathUtilities.cpp Mon Feb 09 16:05:32 2009 +0000 +++ b/maths/MathUtilities.cpp Tue Feb 10 12:52:43 2009 +0000 @@ -144,6 +144,20 @@ return retVal; } +double MathUtilities::mean(const std::vector &src, + unsigned int start, + unsigned int count) +{ + double sum = 0.; + + for (int i = 0; i < count; ++i) + { + sum += src[start + i]; + } + + return sum / count; +} + void MathUtilities::getFrameMinMax(const double *data, unsigned int len, double *min, double *max) { unsigned int i; @@ -189,7 +203,33 @@ } - *pMax = max; + if (pMax) *pMax = max; + + + return index; +} + +int MathUtilities::getMax( const std::vector & data, double* pMax ) +{ + unsigned int index = 0; + unsigned int i; + double temp = 0.0; + + double max = data[0]; + + for( i = 0; i < data.size(); i++) + { + temp = data[ i ]; + + if( temp > max ) + { + max = temp ; + index = i; + } + + } + + if (pMax) *pMax = max; return index; @@ -289,5 +329,28 @@ } } +void MathUtilities::adaptiveThreshold(std::vector &data) +{ + int sz = int(data.size()); + if (sz == 0) return; + + std::vector smoothed(sz); + + int p_pre = 8; + int p_post = 7; + + for (int i = 0; i < sz; ++i) { + + int first = std::max(0, i - p_pre); + int last = std::min(sz - 1, i + p_post); + + smoothed[i] = mean(data, first, last - first + 1); + } + + for (int i = 0; i < sz; i++) { + data[i] -= smoothed[i]; + if (data[i] < 0.0) data[i] = 0.0; + } +} diff -r 833ca65b0820 -r c8908cdc8c32 maths/MathUtilities.h --- a/maths/MathUtilities.h Mon Feb 09 16:05:32 2009 +0000 +++ b/maths/MathUtilities.h Tue Feb 10 12:52:43 2009 +0000 @@ -21,6 +21,8 @@ static void getFrameMinMax( const double* data, unsigned int len, double* min, double* max ); static double mean( const double* src, unsigned int len ); + static double mean( const std::vector &data, + unsigned int start, unsigned int count ); static double sum( const double* src, unsigned int len ); static double median( const double* src, unsigned int len ); @@ -32,7 +34,8 @@ static void circShift( double* data, int length, int shift); - static int getMax( double* data, unsigned int length, double* max ); + static int getMax( double* data, unsigned int length, double* max = 0 ); + static int getMax( const std::vector &data, double* max = 0 ); static int compareInt(const void * a, const void * b); enum NormaliseType { @@ -46,6 +49,9 @@ static void normalise(std::vector &data, NormaliseType n = NormaliseUnitMax); + + // moving mean threshholding: + static void adaptiveThreshold(std::vector &data); }; #endif diff -r 833ca65b0820 -r c8908cdc8c32 qm-dsp.pro --- a/qm-dsp.pro Mon Feb 09 16:05:32 2009 +0000 +++ b/qm-dsp.pro Tue Feb 10 12:52:43 2009 +0000 @@ -48,6 +48,7 @@ dsp/signalconditioning/Filter.h \ dsp/signalconditioning/FiltFilt.h \ dsp/signalconditioning/Framer.h \ + dsp/tempotracking/DownBeat.h \ dsp/tempotracking/TempoTrack.h \ dsp/tempotracking/TempoTrackV2.h \ dsp/tonal/ChangeDetectionFunction.h \ @@ -83,6 +84,7 @@ dsp/signalconditioning/Filter.cpp \ dsp/signalconditioning/FiltFilt.cpp \ dsp/signalconditioning/Framer.cpp \ + dsp/tempotracking/DownBeat.cpp \ dsp/tempotracking/TempoTrack.cpp \ dsp/tempotracking/TempoTrackV2.cpp \ dsp/tonal/ChangeDetectionFunction.cpp \