Mercurial > hg > match-vamp
changeset 56:506f0d9880db refactors
Merge from default branch
| author | Chris Cannam |
|---|---|
| date | Fri, 14 Nov 2014 10:25:40 +0000 |
| parents | 331a17753663 (diff) a68204b9a529 (current diff) |
| children | bbdcf79f3916 19a93b15fcc3 |
| files | src/Finder.cpp |
| diffstat | 13 files changed, 715 insertions(+), 718 deletions(-) [+] |
line wrap: on
line diff
--- a/Makefile.linux Fri Nov 14 10:24:13 2014 +0000 +++ b/Makefile.linux Fri Nov 14 10:25:40 2014 +0000 @@ -1,5 +1,7 @@ -CXXFLAGS += -fPIC -ffast-math -O3 -Wall -Werror +CXXFLAGS += -fPIC -ffast-math -O3 -Wall -Werror -Wfloat-conversion +#CXXFLAGS += -fPIC -g -Wall -Werror + LDFLAGS += -shared -Wl,-Bstatic -lvamp-sdk -Wl,-Bdynamic -Wl,-Bsymbolic -Wl,-z,defs -lpthread -Wl,--version-script=vamp-plugin.map include Makefile.inc
--- a/src/DistanceMetric.cpp Fri Nov 14 10:24:13 2014 +0000 +++ b/src/DistanceMetric.cpp Fri Nov 14 10:25:40 2014 +0000 @@ -56,12 +56,3 @@ return d / sum * weight; } -int -DistanceMetric::calcDistanceScaled(const vector<double> &f1, - const vector<double> &f2, - double scale) -{ - double distance = calcDistance(f1, f2); - return int(distance * scale); -} -
--- a/src/DistanceMetric.h Fri Nov 14 10:24:13 2014 +0000 +++ b/src/DistanceMetric.h Fri Nov 14 10:25:40 2014 +0000 @@ -51,21 +51,6 @@ double calcDistance(const std::vector<double> &f1, const std::vector<double> &f2); - /** Calculates the Manhattan distance between two vectors, with an - * optional normalisation by the combined values in the - * vectors. Since the vectors contain energy, this could be - * considered as a squared Euclidean distance metric. Note that - * normalisation assumes the values are all non-negative. - * - * @param f1 one of the vectors involved in the distance calculation - * @param f2 one of the vectors involved in the distance calculation - * @param scale the scaling factor to place the result in integer range - * @return the distance, scaled by scale and truncated to an integer - */ - int calcDistanceScaled(const std::vector<double> &f1, - const std::vector<double> &f2, - double scale); - private: DistanceNormalisation m_norm; };
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/FeatureExtractor.cpp Fri Nov 14 10:25:40 2014 +0000 @@ -0,0 +1,156 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Vamp feature extraction plugin using the MATCH audio alignment + algorithm. + + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2007 Simon Dixon, Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "FeatureExtractor.h" + +#include <iostream> + +#include <cstdlib> +#include <cassert> +#include <cmath> + +using namespace std; + +FeatureExtractor::FeatureExtractor(Parameters parameters) : + m_params(parameters), + m_ltAverage(0) +{ + if (m_params.useChromaFrequencyMap) { + m_featureSize = 13; + } else { + m_featureSize = 84; + } + + m_prevFrame = vector<double>(m_featureSize, 0.0); + + makeFreqMap(); +} + +void +FeatureExtractor::makeFreqMap() +{ + m_freqMap = vector<int>(m_params.fftSize / 2 + 1, 0); + + if (m_params.useChromaFrequencyMap) { +#ifdef DEBUG_MATCHER + cerr << "makeFreqMap: calling makeChromaFrequencyMap" << endl; +#endif + makeChromaFrequencyMap(); + } else { +#ifdef DEBUG_MATCHER + cerr << "makeFreqMap: calling makeStandardFrequencyMap" << endl; +#endif + makeStandardFrequencyMap(); + } +} + +void +FeatureExtractor::makeStandardFrequencyMap() +{ + double binWidth = m_params.sampleRate / m_params.fftSize; + int crossoverBin = (int)(2 / (pow(2, 1/12.0) - 1)); + int crossoverMidi = lrint(log(crossoverBin*binWidth/440.0)/ + log(2.0) * 12 + 69); + + // freq = 440 * Math.pow(2, (midi-69)/12.0) / binWidth; + + int i = 0; + while (i <= crossoverBin) { + m_freqMap[i] = i; + ++i; + } + + while (i <= m_params.fftSize/2) { + double midi = log(i*binWidth/440.0) / log(2.0) * 12 + 69; + if (midi > 127) midi = 127; + int target = crossoverBin + lrint(midi) - crossoverMidi; + if (target >= m_featureSize) target = m_featureSize - 1; + m_freqMap[i++] = target; + } +} + +void +FeatureExtractor::makeChromaFrequencyMap() +{ + double binWidth = m_params.sampleRate / m_params.fftSize; + int crossoverBin = (int)(1 / (pow(2, 1/12.0) - 1)); + int i = 0; + while (i <= crossoverBin) { + m_freqMap[i++] = 0; + } + while (i <= m_params.fftSize/2) { + double midi = log(i*binWidth/440.0) / log(2.0) * 12 + 69; + m_freqMap[i++] = (lrint(midi)) % 12 + 1; + } +} + +vector<double> +FeatureExtractor::process(const vector<double> &real, const vector<double> &imag) +{ + vector<double> frame(m_featureSize, 0.0); + + double rms = 0; + for (int i = 0; i <= m_params.fftSize/2; i++) { + double mag = real[i] * real[i] + imag[i] * imag[i]; + rms += mag; + frame[m_freqMap[i]] += mag; + } + rms = sqrt(rms / (m_params.fftSize/2)); + + vector<double> feature(m_featureSize, 0.0); + + double totalEnergy = 0; + if (m_params.useSpectralDifference) { + for (int i = 0; i < m_featureSize; i++) { + totalEnergy += frame[i]; + if (frame[i] > m_prevFrame[i]) { + feature[i] = frame[i] - m_prevFrame[i]; + } else { + feature[i] = 0; + } + } + } else { + for (int i = 0; i < m_featureSize; i++) { + feature[i] = frame[i]; + totalEnergy += feature[i]; + } + } + + if (m_ltAverage == 0) { + m_ltAverage = totalEnergy; + } else { + double decay = m_params.decay; + m_ltAverage = m_ltAverage * decay + totalEnergy * (1.0 - decay); + } + + if (rms <= m_params.silenceThreshold) { + for (int i = 0; i < m_featureSize; i++) { + feature[i] = 0; + } + } else if (m_params.frameNorm == NormaliseFrameToSum1) { + for (int i = 0; i < m_featureSize; i++) { + feature[i] /= totalEnergy; + } + } else if (m_params.frameNorm == NormaliseFrameToLTAverage) { + for (int i = 0; i < m_featureSize; i++) { + feature[i] /= m_ltAverage; + } + } + + m_prevFrame = frame; + return feature; +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/FeatureExtractor.h Fri Nov 14 10:25:40 2014 +0000 @@ -0,0 +1,167 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Vamp feature extraction plugin using the MATCH audio alignment + algorithm. + + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2007 Simon Dixon, Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef FEATURE_EXTRACTOR_H +#define FEATURE_EXTRACTOR_H + +#include <vector> + +/** + * Convert frequency-domain audio frames into features suitable for + * MATCH alignment calculation. The default feature is a warping of + * the frequency data to map higher frequencies into a linear scale. A + * chroma mapping is also available. + * + * Note that FeatureExtractor maintains internal frame-to-frame state: + * use one FeatureExtractor per audio source, and construct a new one + * for each new source. + */ +class FeatureExtractor +{ +public: + enum FrameNormalisation { + + /** Do not normalise frames */ + NoFrameNormalisation, + + /** Normalise each frame to have a sum of 1 */ + NormaliseFrameToSum1, + + /** Normalise each frame by the long-term average of the + * summed energy */ + NormaliseFrameToLTAverage, + }; + + struct Parameters { + + Parameters(float rate_, int fftSize_) : + sampleRate(rate_), + frameNorm(NormaliseFrameToSum1), + useSpectralDifference(true), + useChromaFrequencyMap(false), + fftSize(fftSize_), + silenceThreshold(0.01), + decay(0.99) + {} + + /** Sample rate of audio */ + float sampleRate; + + /** Type of audio frame normalisation */ + FrameNormalisation frameNorm; + + /** Flag indicating whether or not the half-wave rectified + * spectral difference should be used in calculating the + * distance metric for pairs of audio frames, instead of the + * straight spectrum values. */ + bool useSpectralDifference; + + /** Flag indicating whether to use a chroma frequency map (12 + * bins) instead of the default warped spectrogram */ + bool useChromaFrequencyMap; + + /** Spacing of audio frames (determines the amount of overlap or + * skip between frames). This value is expressed in + * seconds. */ + double hopTime; + + /** Size of an FFT frame in samples. Note that the data passed + * in is already in the frequency domain, so this expresses + * the size of the frame that the caller will be providing. */ + int fftSize; + + /** RMS level below which frame is considered silent */ + double silenceThreshold; + + /** Frame-to-frame decay factor in calculating long-term average */ + double decay; + }; + + /** + * Construct a FeatureExtractor with the given parameters. + * + * Note that FeatureExtractor maintains internal frame-to-frame + * state: use one FeatureExtractor per audio source, and construct + * a new one for each new source. + */ + FeatureExtractor(Parameters params); + + /** + * Return the feature vector size that will be returned from process(). + */ + int getFeatureSize() const { return m_featureSize; } + + /** + * Process one frequency-domain audio frame (provided as real & + * imaginary components from the FFT output). Return a feature + * vector of size given by getFeatureSize(). Input vectors must + * have at least params.fftSize/2+1 elements each. + * + * Operates by mapping the frequency bins into a part-linear + * part-logarithmic array, then (optionally) computing the + * half-wave rectified spectral difference from the previous + * frame, then (optionally) normalising to a sum of 1. + * + * Return value is the frame (post-processed, with warping, + * rectification, and normalisation as appropriate). + */ + std::vector<double> process(const std::vector<double> &real, + const std::vector<double> &imag); + +protected: + /** Make either standard or chroma map, depending on m_params */ + void makeFreqMap(); + + /** Creates a map of FFT frequency bins to comparison bins. Where + * the spacing of FFT bins is less than 0.5 semitones, the + * mapping is one to one. Where the spacing is greater than 0.5 + * semitones, the FFT energy is mapped into semitone-wide + * bins. No scaling is performed; that is the energy is summed + * into the comparison bins. */ + void makeStandardFrequencyMap(); + + /** Creates a map of FFT frequency bins to semitone chroma bins. */ + void makeChromaFrequencyMap(); + + /** Configuration parameters */ + Parameters m_params; + + /** Long term average frame energy (in frequency domain + * representation). */ + double m_ltAverage; + + /** A mapping function for mapping FFT bins to final frequency + * bins. The mapping is linear (1-1) until the resolution + * reaches 2 points per semitone, then logarithmic with a + * semitone resolution. e.g. for 44.1kHz sampling rate and + * fftSize of 2048 (46ms), bin spacing is 21.5Hz, which is mapped + * linearly for bins 0-34 (0 to 732Hz), and logarithmically for + * the remaining bins (midi notes 79 to 127, bins 35 to 83), + * where all energy above note 127 is mapped into the final + * bin. */ + std::vector<int> m_freqMap; + + /** The size of a returned feature. */ + int m_featureSize; + + /** The most recent frame; used for calculating the frame to frame + * spectral difference. This is therefore frequency warped but + * not yet normalised. */ + std::vector<double> m_prevFrame; +}; + +#endif +
--- a/src/Finder.cpp Fri Nov 14 10:24:13 2014 +0000 +++ b/src/Finder.cpp Fri Nov 14 10:25:40 2014 +0000 @@ -23,7 +23,7 @@ Finder::Finder(Matcher *p1, Matcher *p2) { - if (!p1->firstPM) + if (!p1->m_firstPM) std::cerr << "Warning: wrong args in Finder()" << std::endl; pm1 = p1; pm2 = p2; @@ -44,42 +44,42 @@ { if (i1 >= 0) { index1 = i1; - index2 = i2 - pm1->first[i1]; + index2 = i2 - pm1->m_first[i1]; } - return (i1 >= 0) && (i2 >= pm1->first[i1]) && (i2 < pm1->last[i1]); + return (i1 >= 0) && (i2 >= pm1->m_first[i1]) && (i2 < pm1->m_last[i1]); } // find() void Finder::getColRange(int row, int *range) { - range[0] = pm1->first[row]; - range[1] = pm1->last[row]; + range[0] = pm1->m_first[row]; + range[1] = pm1->m_last[row]; } // getColRange() void Finder::getRowRange(int col, int *range) { - range[0] = pm2->first[col]; - range[1] = pm2->last[col]; + range[0] = pm2->m_first[col]; + range[1] = pm2->m_last[col]; } // getRowRange() -int +Matcher::Advance Finder::getExpandDirection(int row, int col) { return getExpandDirection(row, col, false); } // getExpandDirection() -int +Matcher::Advance Finder::getExpandDirection(int row, int col, bool check) { - int min = getPathCost(row, col); + double min = getPathCost(row, col); bestRow = row; bestCol = col; getRowRange(col, rowRange); if (rowRange[1] > row+1) rowRange[1] = row+1; // don't cheat by looking at future :) for (int index = rowRange[0]; index < rowRange[1]; index++) { - int tmp = getPathCost(index, col); + double tmp = getPathCost(index, col); if (tmp < min) { min = tmp; bestRow = index; @@ -89,7 +89,7 @@ if (colRange[1] > col+1) colRange[1] = col+1; // don't cheat by looking at future :) for (int index = colRange[0]; index < colRange[1]; index++) { - int tmp = getPathCost(row, index); + double tmp = getPathCost(row, index); if (tmp < min) { min = tmp; bestCol = index; @@ -100,88 +100,111 @@ // System.err.println(" " + pm1->frameCount + " " + pm2->frameCount); if (check) { // System.err.println(find(row+1, col) + " " + find(row, col+1)); - if (!find(row, col+1)) - return ADVANCE_THIS; - if (!find(row+1, col)) - return ADVANCE_OTHER; + if (!find(row, col+1)) { + return Matcher::AdvanceThis; + } else if (!find(row+1, col)) { + return Matcher::AdvanceOther; + } } - return ((bestRow==row)? ADVANCE_THIS: 0) | - ((bestCol==col)? ADVANCE_OTHER: 0); + + if (bestRow == row) { + if (bestCol == col) { + return Matcher::AdvanceBoth; + } else { + return Matcher::AdvanceThis; + } + } else if (bestCol == col) { + return Matcher::AdvanceOther; + } else { + return Matcher::AdvanceNone; + } } // getExpandDirection() -unsigned char +float Finder::getDistance(int row, int col) { if (find(row, col)) { - return pm1->distance[row][col - pm1->first[row]]; + return pm1->m_distance[row][col - pm1->m_first[row]]; } std::cerr << "getDistance(" << row << "," << col << "): out of bounds" << std::endl; throw "getDistance index out of bounds"; } // getDistance()/2 void -Finder::setDistance(int row, int col, unsigned char b) +Finder::setDistance(int row, int col, float b) { if (find(row, col)) { - pm1->distance[row][col - pm1->first[row]] = b; + pm1->m_distance[row][col - pm1->m_first[row]] = b; return; } std::cerr << "setDistance(" << row << "," << col << "," << b << "): out of bounds" << std::endl; throw "setDistance index out of bounds"; } // setDistance() -int +double Finder::getPathCost(int row, int col) { if (find(row, col)) // "1" avoids div by 0 below - return pm1->bestPathCost[row][col - pm1->first[row]]*100/ (1+row+col); + return pm1->m_bestPathCost[row][col - pm1->m_first[row]] / float(1+row+col); std::cerr << "getPathCost(" << row << "," << col << "): out of bounds" << std::endl; throw "getPathCost index out of bounds"; } // getPathCost() -int +double Finder::getRawPathCost(int row, int col) { if (find(row, col)) - return pm1->bestPathCost[row][col - pm1->first[row]]; + return pm1->m_bestPathCost[row][col - pm1->m_first[row]]; std::cerr << "getRawPathCost(" << row << "," << col << "): out of bounds" << std::endl; throw "getRawPathCost index out of bounds"; } // getRawPathCost() void -Finder::setPathCost(int row, int col, int i) +Finder::setPathCost(int row, int col, double cost) { if (find(row, col)) { - pm1->bestPathCost[row][col - pm1->first[row]] = i; + pm1->m_bestPathCost[row][col - pm1->m_first[row]] = cost; return; } - std::cerr << "setPathCost(" << row << "," << col << "," << i << "): out of bounds" << std::endl; + std::cerr << "setPathCost(" << row << "," << col << "," << cost << "): out of bounds" << std::endl; throw "setPathCost index out of bounds"; } // setPathCost() -unsigned char +Matcher::Advance +Finder::getAdvance() +{ + return pm1->m_advance[index1][index2]; +} + +void +Finder::setAdvance(Matcher::Advance a) +{ + pm1->m_advance[index1][index2] = a; +} + +float Finder::getDistance() { - return pm1->distance[index1][index2]; + return pm1->m_distance[index1][index2]; } // getDistance()/0 void -Finder::setDistance(int b) +Finder::setDistance(float b) { - pm1->distance[index1][index2] = (unsigned char)b; + pm1->m_distance[index1][index2] = b; } // setDistance() -int +double Finder::getPathCost() { - return pm1->bestPathCost[index1][index2]; + return pm1->m_bestPathCost[index1][index2]; } // getPathCost() void -Finder::setPathCost(int i) +Finder::setPathCost(double cost) { - pm1->bestPathCost[index1][index2] = i; + pm1->m_bestPathCost[index1][index2] = cost; } // setPathCost() void @@ -194,50 +217,47 @@ int thisRowStart, c; int prevRowStart = 0, prevRowStop = 0; for (int r = r1; r <= r2; r++) { - thisRowStart = pm1->first[r]; + thisRowStart = pm1->m_first[r]; if (thisRowStart < c1) thisRowStart = c1; for (c = thisRowStart; c <= c2; c++) { if (find(r,c)) { int i2 = index2; - int newCost = pm1->distance[r][i2]; - int dir = 0; + float newCost = pm1->m_distance[r][i2]; + Matcher::Advance dir = Matcher::AdvanceNone; if (r > r1) { // not first row - int min = -1; + double min = -1; if ((c > prevRowStart) && (c <= prevRowStop)) { // diagonal from (r-1,c-1) - min = pm1->bestPathCost[r-1][c-pm1->first[r-1]-1] + + min = pm1->m_bestPathCost[r-1][c-pm1->m_first[r-1]-1] + newCost * 2; - dir = ADVANCE_BOTH; + dir = Matcher::AdvanceBoth; } if ((c >= prevRowStart) && (c < prevRowStop)) { // vertical from (r-1,c) - int cost = pm1->bestPathCost[r-1][c-pm1->first[r-1]] + + double cost = pm1->m_bestPathCost[r-1][c-pm1->m_first[r-1]] + newCost; if ((min == -1) || (cost < min)) { min = cost; - dir = ADVANCE_THIS; + dir = Matcher::AdvanceThis; } } if (c > thisRowStart) { // horizontal from (r,c-1) - int cost =pm1->bestPathCost[r][i2-1]+newCost; + double cost =pm1->m_bestPathCost[r][i2-1]+newCost; if ((min == -1) || (cost < min)) { min = cost; - dir = ADVANCE_OTHER; + dir = Matcher::AdvanceOther; } } - pm1->bestPathCost[r][i2] = min; + pm1->m_bestPathCost[r][i2] = min; } else if (c > thisRowStart) { // first row // horizontal from (r,c-1) - pm1->bestPathCost[r][i2] = pm1->bestPathCost[r][i2-1] + + pm1->m_bestPathCost[r][i2] = pm1->m_bestPathCost[r][i2-1] + newCost; - dir = ADVANCE_OTHER; + dir = Matcher::AdvanceOther; } - if ((r != r1) || (c != c1)) { - pm1->distance[r][i2] = (unsigned char) - ((pm1->distance[r][i2] & MASK) | dir); - } + pm1->m_advance[r][i2] = dir; } else break; // end of row } @@ -264,21 +284,21 @@ pathx.push_back(x); pathy.push_back(y); - switch (getDistance() & ADVANCE_BOTH) { - case ADVANCE_THIS: + switch (getAdvance()) { + case Matcher::AdvanceThis: // cerr << ", going down (dist = " << (int)getDistance() << ")" << endl; y--; break; - case ADVANCE_OTHER: + case Matcher::AdvanceOther: // cerr << ", going left (dist = " << (int)getDistance() << ")" << endl; x--; break; - case ADVANCE_BOTH: + case Matcher::AdvanceBoth: // cerr << ", going diag (dist = " << (int)getDistance() << ")" << endl; x--; y--; break; - default: // this would indicate a bug, but we wouldn't want to hang + case Matcher::AdvanceNone: // this would indicate a bug, but we wouldn't want to hang // cerr << "WARNING: Neither matcher advanced in path backtrack at (" << x << "," << y << ")" << endl; if (x > y) { x--;
--- a/src/Finder.h Fri Nov 14 10:24:13 2014 +0000 +++ b/src/Finder.h Fri Nov 14 10:25:40 2014 +0000 @@ -57,21 +57,24 @@ * column. */ void getRowRange(int col, int *range); - int getExpandDirection(int row, int col); - int getExpandDirection(int row, int col, bool check); + Matcher::Advance getExpandDirection(int row, int col); + Matcher::Advance getExpandDirection(int row, int col, bool check); - unsigned char getDistance(int row, int col); - void setDistance(int row, int col, unsigned char b); + float getDistance(int row, int col); + void setDistance(int row, int col, float b); - int getPathCost(int row, int col); - int getRawPathCost(int row, int col); - void setPathCost(int row, int col, int i); + double getPathCost(int row, int col); + double getRawPathCost(int row, int col); //!!!??? + void setPathCost(int row, int col, double i); - unsigned char getDistance(); - void setDistance(int b); + Matcher::Advance getAdvance(); + void setAdvance(Matcher::Advance a); + + float getDistance(); + void setDistance(float b); - int getPathCost(); - void setPathCost(int i); + double getPathCost(); + void setPathCost(double i); /** Calculates a rectangle of the path cost matrix so that the * minimum cost path between the bottom left and top right
--- a/src/MatchFeatureFeeder.cpp Fri Nov 14 10:24:13 2014 +0000 +++ b/src/MatchFeatureFeeder.cpp Fri Nov 14 10:25:40 2014 +0000 @@ -32,6 +32,12 @@ void MatchFeatureFeeder::feed(vector<double> f1, vector<double> f2) { + // We maintain two FIFO queues of feature vectors, one per input + // stream. When the match-feeder function is entered, it knows + // that it has at least one feature in each queue. It loops, + // processing up to one feature per matcher, until a queue is + // empty. Then it returns, to be called again with more data. + q1.push(f1); q2.push(f2); @@ -43,27 +49,30 @@ void MatchFeatureFeeder::feedBlock() { - if (pm1->frameCount < pm1->blockSize) { // fill initial block + if (pm1->m_frameCount < pm1->m_blockSize) { // fill initial block feed1(); feed2(); } - else if (pm1->runCount >= pm1->params.maxRunCount) { // slope constraints + else if (pm1->m_runCount >= pm1->m_params.maxRunCount) { // slope constraints feed2(); - } else if (pm2->runCount >= pm2->params.maxRunCount) { + } else if (pm2->m_runCount >= pm2->m_params.maxRunCount) { feed1(); } else { switch (finder->getExpandDirection - (pm1->frameCount-1, pm2->frameCount-1)) { - case ADVANCE_THIS: + (pm1->m_frameCount-1, pm2->m_frameCount-1)) { + case Matcher::AdvanceThis: feed1(); break; - case ADVANCE_OTHER: + case Matcher::AdvanceOther: feed2(); break; - case ADVANCE_BOTH: + case Matcher::AdvanceBoth: feed1(); feed2(); break; + case Matcher::AdvanceNone: + cerr << "finder says AdvanceNone!" << endl; + break; } } }
--- a/src/MatchFeeder.cpp Fri Nov 14 10:24:13 2014 +0000 +++ b/src/MatchFeeder.cpp Fri Nov 14 10:25:40 2014 +0000 @@ -21,7 +21,7 @@ MatchFeeder::MatchFeeder(Matcher *m1, Matcher *m2) : pm1(m1), pm2(m2) { - fftSize = m1->params.fftSize; + fftSize = m1->m_params.fftSize; finder = new Finder(m1, m2); reBuffer = new double[fftSize/2+1]; imBuffer = new double[fftSize/2+1]; @@ -97,7 +97,7 @@ Features ff; vector<double> f1, f2; - if (pm1->frameCount < pm1->blockSize) { // fill initial block + if (pm1->m_frameCount < pm1->m_blockSize) { // fill initial block // std::cerr << "feeding initial block" << std::endl; f1 = feed1(); f2 = feed2(); @@ -106,28 +106,28 @@ // feed2(); //!!! } else if (pm2->atEnd) // feed1(); - else if (pm1->runCount >= pm1->params.maxRunCount) { // slope constraints + else if (pm1->m_runCount >= pm1->m_params.maxRunCount) { // slope constraints // std::cerr << "pm1 too slopey" << std::endl; f2 = feed2(); - } else if (pm2->runCount >= pm2->params.maxRunCount) { + } else if (pm2->m_runCount >= pm2->m_params.maxRunCount) { // std::cerr << "pm2 too slopey" << std::endl; f1 = feed1(); } else { switch (finder->getExpandDirection - (pm1->frameCount-1, pm2->frameCount-1)) { - case ADVANCE_THIS: -// std::cerr << "finder says ADVANCE_THIS" << std::endl; + (pm1->m_frameCount-1, pm2->m_frameCount-1)) { + case Matcher::AdvanceThis: f1 = feed1(); break; - case ADVANCE_OTHER: -// std::cerr << "finder says ADVANCE_OTHER" << std::endl; + case Matcher::AdvanceOther: f2 = feed2(); break; - case ADVANCE_BOTH: -// std::cerr << "finder says ADVANCE_BOTH" << std::endl; + case Matcher::AdvanceBoth: f1 = feed1(); f2 = feed2(); break; + case Matcher::AdvanceNone: + cerr << "finder says AdvanceNone!" << endl; + break; } }
--- a/src/MatchVampPlugin.cpp Fri Nov 14 10:24:13 2014 +0000 +++ b/src/MatchVampPlugin.cpp Fri Nov 14 10:25:40 2014 +0000 @@ -45,11 +45,11 @@ // sample rates static float sampleRateMin = 5000.f; -static float defaultStepTime = 0.020; +static float defaultStepTime = 0.020f; MatchVampPlugin::MatchVampPlugin(float inputSampleRate) : Plugin(inputSampleRate), - m_stepSize(inputSampleRate * defaultStepTime + 0.001), + m_stepSize(int(inputSampleRate * defaultStepTime + 0.001)), m_stepTime(defaultStepTime), m_blockSize(2048), m_serialise(false), @@ -57,7 +57,9 @@ m_locked(false), m_smooth(true), m_params(inputSampleRate, defaultStepTime, m_blockSize), - m_defaultParams(inputSampleRate, defaultStepTime, m_blockSize) + m_defaultParams(inputSampleRate, defaultStepTime, m_blockSize), + m_feParams(inputSampleRate, m_blockSize), + m_defaultFeParams(inputSampleRate, m_blockSize) { if (inputSampleRate < sampleRateMin) { std::cerr << "MatchVampPlugin::MatchVampPlugin: input sample rate " @@ -157,7 +159,7 @@ desc.description = "Type of normalisation to use for frequency-domain audio features"; desc.minValue = 0; desc.maxValue = 2; - desc.defaultValue = (int)m_defaultParams.frameNorm; + desc.defaultValue = (int)m_defaultFeParams.frameNorm; desc.isQuantized = true; desc.quantizeStep = 1; desc.valueNames.clear(); @@ -187,7 +189,7 @@ desc.description = "Whether to use half-wave rectified spectral difference instead of straight spectrum"; desc.minValue = 0; desc.maxValue = 1; - desc.defaultValue = m_defaultParams.useSpectralDifference ? 1 : 0; + desc.defaultValue = m_defaultFeParams.useSpectralDifference ? 1 : 0; desc.isQuantized = true; desc.quantizeStep = 1; list.push_back(desc); @@ -197,7 +199,7 @@ desc.description = "Whether to use a chroma frequency map instead of the default warped spectrogram"; desc.minValue = 0; desc.maxValue = 1; - desc.defaultValue = m_defaultParams.useChromaFrequencyMap ? 1 : 0; + desc.defaultValue = m_defaultFeParams.useChromaFrequencyMap ? 1 : 0; desc.isQuantized = true; desc.quantizeStep = 1; list.push_back(desc); @@ -217,7 +219,7 @@ desc.description = "Width of the search zone (error margin) either side of the ongoing match position, in seconds"; desc.minValue = 1; desc.maxValue = 60; - desc.defaultValue = m_defaultParams.blockTime; + desc.defaultValue = (float)m_defaultParams.blockTime; desc.isQuantized = true; desc.quantizeStep = 1; desc.unit = "s"; @@ -243,17 +245,17 @@ if (name == "serialise") { return m_serialise ? 1.0 : 0.0; } else if (name == "framenorm") { - return (int)m_params.frameNorm; + return (int)m_feParams.frameNorm; } else if (name == "distnorm") { return (int)m_params.distanceNorm; } else if (name == "usespecdiff") { - return m_params.useSpectralDifference ? 1.0 : 0.0; + return m_feParams.useSpectralDifference ? 1.0 : 0.0; } else if (name == "usechroma") { - return m_params.useChromaFrequencyMap ? 1.0 : 0.0; + return m_feParams.useChromaFrequencyMap ? 1.0 : 0.0; } else if (name == "gradientlimit") { return m_params.maxRunCount; } else if (name == "zonewidth") { - return m_params.blockTime; + return (float)m_params.blockTime; } else if (name == "smooth") { return m_smooth ? 1.0 : 0.0; } @@ -267,13 +269,13 @@ if (name == "serialise") { m_serialise = (value > 0.5); } else if (name == "framenorm") { - m_params.frameNorm = (Matcher::FrameNormalisation)(int(value + 0.1)); + m_feParams.frameNorm = (FeatureExtractor::FrameNormalisation)(int(value + 0.1)); } else if (name == "distnorm") { m_params.distanceNorm = (DistanceMetric::DistanceNormalisation)(int(value + 0.1)); } else if (name == "usespecdiff") { - m_params.useSpectralDifference = (value > 0.5); + m_feParams.useSpectralDifference = (value > 0.5); } else if (name == "usechroma") { - m_params.useChromaFrequencyMap = (value > 0.5); + m_feParams.useChromaFrequencyMap = (value > 0.5); } else if (name == "gradientlimit") { m_params.maxRunCount = int(value + 0.1); } else if (name == "zonewidth") { @@ -286,7 +288,7 @@ size_t MatchVampPlugin::getPreferredStepSize() const { - return m_inputSampleRate * defaultStepTime; + return int(m_inputSampleRate * defaultStepTime + 0.001); } size_t @@ -300,8 +302,9 @@ { m_params.hopTime = m_stepTime; m_params.fftSize = m_blockSize; - pm1 = new Matcher(m_params, 0); - pm2 = new Matcher(m_params, pm1); + m_feParams.fftSize = m_blockSize; + pm1 = new Matcher(m_params, m_feParams, 0); + pm2 = new Matcher(m_params, m_feParams, pm1); pm1->setOtherMatcher(pm2); feeder = new MatchFeeder(pm1, pm2); } @@ -351,7 +354,7 @@ { OutputList list; - float outRate = 1.0 / m_stepTime; + float outRate = 1.0f / m_stepTime; OutputDescriptor desc; desc.identifier = "path"; @@ -420,12 +423,14 @@ m_abRatioOutNo = list.size(); list.push_back(desc); + int featureSize = FeatureExtractor(m_feParams).getFeatureSize(); + desc.identifier = "a_features"; desc.name = "A Features"; desc.description = "Spectral features extracted from performance A"; desc.unit = ""; desc.hasFixedBinCount = true; - desc.binCount = Matcher::getFeatureSizeFor(m_params); + desc.binCount = featureSize; desc.hasKnownExtents = false; desc.isQuantized = false; desc.sampleType = OutputDescriptor::FixedSampleRate; @@ -438,7 +443,7 @@ desc.description = "Spectral features extracted from performance B"; desc.unit = ""; desc.hasFixedBinCount = true; - desc.binCount = Matcher::getFeatureSizeFor(m_params); + desc.binCount = featureSize; desc.hasKnownExtents = false; desc.isQuantized = false; desc.sampleType = OutputDescriptor::FixedSampleRate; @@ -478,7 +483,7 @@ for (int i = 0; i < (int)ff.f1.size(); ++i) { f.values.clear(); for (int j = 0; j < (int)ff.f1[i].size(); ++j) { - f.values.push_back(ff.f1[i][j]); + f.values.push_back(float(ff.f1[i][j])); } returnFeatures[m_aFeaturesOutNo].push_back(f); } @@ -486,7 +491,7 @@ for (int i = 0; i < (int)ff.f2.size(); ++i) { f.values.clear(); for (int j = 0; j < (int)ff.f2[i].size(); ++j) { - f.values.push_back(ff.f2[i][j]); + f.values.push_back(float(ff.f2[i][j])); } returnFeatures[m_bFeaturesOutNo].push_back(f); } @@ -524,7 +529,7 @@ feature.hasTimestamp = true; feature.timestamp = m_startTime + xt; feature.values.clear(); - feature.values.push_back(yt.sec + double(yt.nsec)/1.0e9); + feature.values.push_back(float(yt.sec + double(yt.nsec)/1.0e9)); returnFeatures[m_pathOutNo].push_back(feature); if (x != prevx) { @@ -532,12 +537,12 @@ feature.hasTimestamp = true; feature.timestamp = m_startTime + xt; feature.values.clear(); - feature.values.push_back(yt.sec + yt.msec()/1000.0); + feature.values.push_back(float(yt.sec + yt.msec()/1000.0)); returnFeatures[m_abOutNo].push_back(feature); Vamp::RealTime diff = yt - xt; feature.values.clear(); - feature.values.push_back(diff.sec + diff.msec()/1000.0); + feature.values.push_back(float(diff.sec + diff.msec()/1000.0)); returnFeatures[m_abDivOutNo].push_back(feature); if (i > 0) { @@ -560,7 +565,7 @@ feature.hasTimestamp = true; feature.timestamp = m_startTime + yt; feature.values.clear(); - feature.values.push_back(xt.sec + xt.msec()/1000.0); + feature.values.push_back(float(xt.sec + xt.msec()/1000.0)); returnFeatures[m_baOutNo].push_back(feature); }
--- a/src/MatchVampPlugin.h Fri Nov 14 10:24:13 2014 +0000 +++ b/src/MatchVampPlugin.h Fri Nov 14 10:25:40 2014 +0000 @@ -26,6 +26,7 @@ #endif #include "Matcher.h" +#include "FeatureExtractor.h" class MatchFeeder; @@ -83,6 +84,9 @@ Matcher::Parameters m_params; Matcher::Parameters m_defaultParams; + FeatureExtractor::Parameters m_feParams; + FeatureExtractor::Parameters m_defaultFeParams; + mutable int m_pathOutNo; mutable int m_abOutNo; mutable int m_baOutNo;
--- a/src/Matcher.cpp Fri Nov 14 10:24:13 2014 +0000 +++ b/src/Matcher.cpp Fri Nov 14 10:25:40 2014 +0000 @@ -21,71 +21,56 @@ #include <cstdlib> #include <cassert> -bool Matcher::silent = true; - //#define DEBUG_MATCHER 1 -Matcher::Matcher(Parameters parameters, Matcher *p) : - params(parameters), - metric(parameters.distanceNorm) +Matcher::Matcher(Parameters parameters, + FeatureExtractor::Parameters feParams, + Matcher *p) : + m_params(parameters), + m_featureExtractor(feParams), + m_metric(parameters.distanceNorm) { #ifdef DEBUG_MATCHER - cerr << "Matcher::Matcher(" << params.sampleRate << ", " << p << ")" << endl; + cerr << "Matcher::Matcher(" << m_params.sampleRate << ", " << p << ")" << endl; #endif - otherMatcher = p; // the first matcher will need this to be set later - firstPM = (!p); - ltAverage = 0; - frameCount = 0; - runCount = 0; - freqMapSize = 0; - externalFeatureSize = 0; - featureSize = 0; - blockSize = 0; + m_otherMatcher = p; // the first matcher will need this to be set later + m_firstPM = (!p); + m_frameCount = 0; + m_runCount = 0; + m_featureSize = m_featureExtractor.getFeatureSize(); + m_blockSize = 0; - blockSize = lrint(params.blockTime / params.hopTime); + m_blockSize = lrint(m_params.blockTime / m_params.hopTime); #ifdef DEBUG_MATCHER - cerr << "Matcher: blockSize = " << blockSize << endl; + cerr << "Matcher: m_blockSize = " << m_blockSize << endl; #endif - distance = 0; - bestPathCost = 0; - distYSizes = 0; - distXSize = 0; - - initialised = false; + m_initialised = false; } -Matcher::Matcher(Parameters parameters, Matcher *p, int featureSize) : - params(parameters), - externalFeatureSize(featureSize), - metric(parameters.distanceNorm) +Matcher::Matcher(Parameters parameters, Matcher *p, int m_featureSize_) : + m_params(parameters), + m_featureSize(m_featureSize_), + m_featureExtractor(FeatureExtractor::Parameters(m_params.sampleRate, m_params.fftSize)), // unused default config + m_metric(parameters.distanceNorm) { #ifdef DEBUG_MATCHER - cerr << "Matcher::Matcher(" << params.sampleRate << ", " << p << ", " << featureSize << ")" << endl; + cerr << "Matcher::Matcher(" << m_params.sampleRate << ", " << p << ", " << m_featureSize << ")" << endl; #endif - otherMatcher = p; // the first matcher will need this to be set later - firstPM = (!p); - ltAverage = 0; - frameCount = 0; - runCount = 0; - freqMapSize = 0; - featureSize = 0; - blockSize = 0; + m_otherMatcher = p; // the first matcher will need this to be set later + m_firstPM = (!p); + m_frameCount = 0; + m_runCount = 0; + m_blockSize = 0; - blockSize = lrint(params.blockTime / params.hopTime); + m_blockSize = lrint(m_params.blockTime / m_params.hopTime); #ifdef DEBUG_MATCHER - cerr << "Matcher: blockSize = " << blockSize << endl; + cerr << "Matcher: m_blockSize = " << m_blockSize << endl; #endif - distance = 0; - bestPathCost = 0; - distYSizes = 0; - distXSize = 0; - - initialised = false; - + m_initialised = false; } Matcher::~Matcher() @@ -93,312 +78,113 @@ #ifdef DEBUG_MATCHER cerr << "Matcher(" << this << ")::~Matcher()" << endl; #endif - - if (initialised) { - - for (int i = 0; i < distXSize; ++i) { - if (distance[i]) { - free(distance[i]); - free(bestPathCost[i]); - } - } - free(distance); - free(bestPathCost); - - free(first); - free(last); - - free(distYSizes); - } } void Matcher::init() { - if (initialised) return; + if (m_initialised) return; - initialised = true; + m_frames = vector<vector<double> > + (m_blockSize, vector<double>(m_featureSize, 0)); - if (externalFeatureSize == 0) { - freqMapSize = getFeatureSizeFor(params); - featureSize = freqMapSize; - makeFreqMap(); - } else { - featureSize = externalFeatureSize; - } + m_distXSize = m_blockSize * 2; - initVector<double>(prevFrame, featureSize); - initVector<double>(newFrame, featureSize); - initMatrix<double>(frames, blockSize, featureSize); - initVector<double>(totalEnergies, blockSize); + size(); - int distSize = (params.maxRunCount + 1) * blockSize; - - distXSize = blockSize * 2; - - distance = (unsigned char **)malloc(distXSize * sizeof(unsigned char *)); - bestPathCost = (int **)malloc(distXSize * sizeof(int *)); - distYSizes = (int *)malloc(distXSize * sizeof(int)); - - for (int i = 0; i < blockSize; ++i) { - distance[i] = (unsigned char *)malloc(distSize * sizeof(unsigned char)); - bestPathCost[i] = (int *)malloc(distSize * sizeof(int)); - distYSizes[i] = distSize; - } - for (int i = blockSize; i < distXSize; ++i) { - distance[i] = 0; - } + m_frameCount = 0; + m_runCount = 0; - first = (int *)malloc(distXSize * sizeof(int)); - last = (int *)malloc(distXSize * sizeof(int)); - - frameCount = 0; - runCount = 0; - ltAverage = 0; - -} // init - -void -Matcher::makeFreqMap() -{ - initVector<int>(freqMap, params.fftSize/2 + 1); - - if (params.useChromaFrequencyMap) { -#ifdef DEBUG_MATCHER - cerr << "makeFreqMap: calling makeChromaFrequencyMap" << endl; -#endif - makeChromaFrequencyMap(); - } else { -#ifdef DEBUG_MATCHER - cerr << "makeFreqMap: calling makeStandardFrequencyMap" << endl; -#endif - makeStandardFrequencyMap(); - } -} // makeFreqMap() - -int -Matcher::getFeatureSizeFor(Parameters params) -{ - if (params.useChromaFrequencyMap) { - return 13; - } else { - return 84; - } + m_initialised = true; } void -Matcher::makeStandardFrequencyMap() +Matcher::size() { - double binWidth = params.sampleRate / params.fftSize; - int crossoverBin = (int)(2 / (pow(2, 1/12.0) - 1)); - int crossoverMidi = lrint(log(crossoverBin*binWidth/440.0)/ - log(2.0) * 12 + 69); - // freq = 440 * Math.pow(2, (midi-69)/12.0) / binWidth; - int i = 0; - while (i <= crossoverBin) { - freqMap[i] = i; - ++i; - } - while (i <= params.fftSize/2) { - double midi = log(i*binWidth/440.0) / log(2.0) * 12 + 69; - if (midi > 127) midi = 127; - int target = crossoverBin + lrint(midi) - crossoverMidi; - if (target >= freqMapSize) target = freqMapSize - 1; - freqMap[i++] = target; - } - - if (!silent) { - cerr << "Standard map size: " << freqMapSize - << "; Crossover at: " << crossoverBin << endl; - for (i = 0; i < params.fftSize / 2; i++) - cerr << "freqMap[" << i << "] = " << freqMap[i] << endl; - } -} // makeStandardFrequencyMap() - -void -Matcher::makeChromaFrequencyMap() -{ - double binWidth = params.sampleRate / params.fftSize; - int crossoverBin = (int)(1 / (pow(2, 1/12.0) - 1)); - // freq = 440 * Math.pow(2, (midi-69)/12.0) / binWidth; - int i = 0; - while (i <= crossoverBin) - freqMap[i++] = 0; - while (i <= params.fftSize/2) { - double midi = log(i*binWidth/440.0) / log(2.0) * 12 + 69; - freqMap[i++] = (lrint(midi)) % 12 + 1; - } - if (!silent) { - cerr << "Chroma map size: " << freqMapSize - << "; Crossover at: " << crossoverBin << endl; - for (i = 0; i < params.fftSize / 2; i++) - cerr << "freqMap[" << i << "] = " << freqMap[i] << endl; - } -} // makeChromaFrequencyMap() + int distSize = (m_params.maxRunCount + 1) * m_blockSize; + m_bestPathCost.resize(m_distXSize, vector<double>(distSize, 0)); + m_distance.resize(m_distXSize, vector<float>(distSize, 0)); + m_advance.resize(m_distXSize, vector<Advance>(distSize, AdvanceNone)); + m_distYSizes.resize(m_distXSize, distSize); + m_first.resize(m_distXSize, 0); + m_last.resize(m_distXSize, 0); +} vector<double> Matcher::consumeFrame(double *reBuffer, double *imBuffer) { - if (!initialised) init(); + if (!m_initialised) init(); - vector<double> processedFrame = - processFrameFromFreqData(reBuffer, imBuffer); - + vector<double> real(reBuffer, reBuffer + m_params.fftSize/2 + 1); + vector<double> imag(imBuffer, imBuffer + m_params.fftSize/2 + 1); + vector<double> feature = m_featureExtractor.process(real, imag); + int frameIndex = m_frameCount % m_blockSize; + m_frames[frameIndex] = feature; calcAdvance(); - return processedFrame; + return feature; } void Matcher::consumeFeatureVector(std::vector<double> feature) { - if (!initialised) init(); - int frameIndex = frameCount % blockSize; - frames[frameIndex] = feature; + if (!m_initialised) init(); + int frameIndex = m_frameCount % m_blockSize; + m_frames[frameIndex] = feature; calcAdvance(); } -vector<double> -Matcher::processFrameFromFreqData(double *reBuffer, double *imBuffer) -{ - for (int i = 0; i < (int)newFrame.size(); ++i) { - newFrame[i] = 0; - } - double rms = 0; - for (int i = 0; i <= params.fftSize/2; i++) { - double mag = reBuffer[i] * reBuffer[i] + - imBuffer[i] * imBuffer[i]; - rms += mag; - newFrame[freqMap[i]] += mag; - } - rms = sqrt(rms / (params.fftSize/2)); - - int frameIndex = frameCount % blockSize; - - vector<double> processedFrame(freqMapSize, 0.0); - - double totalEnergy = 0; - if (params.useSpectralDifference) { - for (int i = 0; i < freqMapSize; i++) { - totalEnergy += newFrame[i]; - if (newFrame[i] > prevFrame[i]) { - processedFrame[i] = newFrame[i] - prevFrame[i]; - } else { - processedFrame[i] = 0; - } - } - } else { - for (int i = 0; i < freqMapSize; i++) { - processedFrame[i] = newFrame[i]; - totalEnergy += processedFrame[i]; - } - } - totalEnergies[frameIndex] = totalEnergy; - - double decay = frameCount >= 200 ? 0.99: - (frameCount < 100? 0: (frameCount - 100) / 100.0); - - if (ltAverage == 0) - ltAverage = totalEnergy; - else - ltAverage = ltAverage * decay + totalEnergy * (1.0 - decay); - - if (rms <= params.silenceThreshold) - for (int i = 0; i < freqMapSize; i++) - processedFrame[i] = 0; - else if (params.frameNorm == NormaliseFrameToSum1) - for (int i = 0; i < freqMapSize; i++) - processedFrame[i] /= totalEnergy; - else if (params.frameNorm == NormaliseFrameToLTAverage) - for (int i = 0; i < freqMapSize; i++) - processedFrame[i] /= ltAverage; - - vector<double> tmp = prevFrame; - prevFrame = newFrame; - newFrame = tmp; - - frames[frameIndex] = processedFrame; - - if ((frameCount % 100) == 0) { - if (!silent) { - cerr << "Progress:" << frameCount << " " << ltAverage << endl; - } - } - - return processedFrame; -} - void Matcher::calcAdvance() { - int frameIndex = frameCount % blockSize; + int frameIndex = m_frameCount % m_blockSize; - if (frameCount >= distXSize) { -// std::cerr << "Resizing " << distXSize << " -> " << distXSize * 2 << std::endl; - distXSize *= 2; - distance = (unsigned char **)realloc(distance, distXSize * sizeof(unsigned char *)); - bestPathCost = (int **)realloc(bestPathCost, distXSize * sizeof(int *)); - distYSizes = (int *)realloc(distYSizes, distXSize * sizeof(int)); - first = (int *)realloc(first, distXSize * sizeof(int)); - last = (int *)realloc(last, distXSize * sizeof(int)); - - for (int i = distXSize/2; i < distXSize; ++i) { - distance[i] = 0; - } + if (m_frameCount >= m_distXSize) { + m_distXSize *= 2; + size(); } - if (firstPM && (frameCount >= blockSize)) { + if (m_firstPM && (m_frameCount >= m_blockSize)) { - int len = last[frameCount - blockSize] - - first[frameCount - blockSize]; + int len = m_last[m_frameCount - m_blockSize] - + m_first[m_frameCount - m_blockSize]; - // We need to copy distance[frameCount-blockSize] to - // distance[frameCount], and then truncate - // distance[frameCount-blockSize] to its first len elements. + // We need to copy distance[m_frameCount-m_blockSize] to + // distance[m_frameCount], and then truncate + // distance[m_frameCount-m_blockSize] to its first len elements. // Same for bestPathCost. /* - std::cerr << "Matcher(" << this << "): moving " << distYSizes[frameCount - blockSize] << " from " << frameCount - blockSize << " to " - << frameCount << ", allocating " << len << " for " - << frameCount - blockSize << std::endl; + std::cerr << "Matcher(" << this << "): moving " << distYSizes[m_frameCount - m_blockSize] << " from " << m_frameCount - m_blockSize << " to " + << m_frameCount << ", allocating " << len << " for " + << m_frameCount - m_blockSize << std::endl; */ - distance[frameCount] = distance[frameCount - blockSize]; + m_distance[m_frameCount] = m_distance[m_frameCount - m_blockSize]; + m_distance[m_frameCount - m_blockSize].resize(len, 0); - distance[frameCount - blockSize] = (unsigned char *) - malloc(len * sizeof(unsigned char)); - for (int i = 0; i < len; ++i) { - distance[frameCount - blockSize][i] = - distance[frameCount][i]; - } + m_bestPathCost[m_frameCount] = m_bestPathCost[m_frameCount - m_blockSize]; + m_bestPathCost[m_frameCount - m_blockSize].resize(len, 0); - bestPathCost[frameCount] = bestPathCost[frameCount - blockSize]; - - bestPathCost[frameCount - blockSize] = (int *) - malloc(len * sizeof(int)); - for (int i = 0; i < len; ++i) { - bestPathCost[frameCount - blockSize][i] = - bestPathCost[frameCount][i]; - } - - distYSizes[frameCount] = distYSizes[frameCount - blockSize]; - distYSizes[frameCount - blockSize] = len; + m_advance[m_frameCount] = m_advance[m_frameCount - m_blockSize]; + m_advance[m_frameCount - m_blockSize].resize(len, AdvanceNone); + + m_distYSizes[m_frameCount] = m_distYSizes[m_frameCount - m_blockSize]; + m_distYSizes[m_frameCount - m_blockSize] = len; } - int stop = otherMatcher->frameCount; - int index = stop - blockSize; + int stop = m_otherMatcher->m_frameCount; + int index = stop - m_blockSize; if (index < 0) index = 0; - first[frameCount] = index; - last[frameCount] = stop; + m_first[m_frameCount] = index; + m_last[m_frameCount] = stop; - bool overflow = false; - int mn= -1; - int mx= -1; + float mn= -1; + float mx= -1; for ( ; index < stop; index++) { - int dMN = metric.calcDistanceScaled - (frames[frameIndex], - otherMatcher->frames[index % blockSize], - params.distanceScale); + float dMN = (float) m_metric.calcDistance + (m_frames[frameIndex], + m_otherMatcher->m_frames[index % m_blockSize]); if (mx<0) mx = mn = dMN; @@ -406,105 +192,101 @@ mx = dMN; else if (dMN < mn) mn = dMN; - if (dMN >= 255) { - overflow = true; - dMN = 255; - } - if ((frameCount == 0) && (index == 0)) // first element - setValue(0, 0, 0, 0, dMN); - else if (frameCount == 0) // first row - setValue(0, index, ADVANCE_OTHER, + if ((m_frameCount == 0) && (index == 0)) // first element + setValue(0, 0, AdvanceNone, 0, dMN); + else if (m_frameCount == 0) // first row + setValue(0, index, AdvanceOther, getValue(0, index-1, true), dMN); else if (index == 0) // first column - setValue(frameCount, index, ADVANCE_THIS, - getValue(frameCount - 1, 0, true), dMN); - else if (index == otherMatcher->frameCount - blockSize) { + setValue(m_frameCount, index, AdvanceThis, + getValue(m_frameCount - 1, 0, true), dMN); + else if (index == m_otherMatcher->m_frameCount - m_blockSize) { // missing value(s) due to cutoff // - no previous value in current row (resp. column) // - no diagonal value if prev. dir. == curr. dirn - int min2 = getValue(frameCount - 1, index, true); - // if ((firstPM && (first[frameCount - 1] == index)) || - // (!firstPM && (last[index-1] < frameCount))) - if (first[frameCount - 1] == index) - setValue(frameCount, index, ADVANCE_THIS, min2, dMN); + double min2 = getValue(m_frameCount - 1, index, true); + // if ((m_firstPM && (first[m_frameCount - 1] == index)) || + // (!m_firstPM && (m_last[index-1] < m_frameCount))) + if (m_first[m_frameCount - 1] == index) + setValue(m_frameCount, index, AdvanceThis, min2, dMN); else { - int min1 = getValue(frameCount - 1, index - 1, true); + double min1 = getValue(m_frameCount - 1, index - 1, true); if (min1 + dMN <= min2) - setValue(frameCount, index, ADVANCE_BOTH, min1,dMN); + setValue(m_frameCount, index, AdvanceBoth, min1,dMN); else - setValue(frameCount, index, ADVANCE_THIS, min2,dMN); + setValue(m_frameCount, index, AdvanceThis, min2,dMN); } } else { - int min1 = getValue(frameCount, index-1, true); - int min2 = getValue(frameCount - 1, index, true); - int min3 = getValue(frameCount - 1, index-1, true); + double min1 = getValue(m_frameCount, index-1, true); + double min2 = getValue(m_frameCount - 1, index, true); + double min3 = getValue(m_frameCount - 1, index-1, true); if (min1 <= min2) { if (min3 + dMN <= min1) - setValue(frameCount, index, ADVANCE_BOTH, min3,dMN); + setValue(m_frameCount, index, AdvanceBoth, min3,dMN); else - setValue(frameCount, index, ADVANCE_OTHER,min1,dMN); + setValue(m_frameCount, index, AdvanceOther,min1,dMN); } else { if (min3 + dMN <= min2) - setValue(frameCount, index, ADVANCE_BOTH, min3,dMN); + setValue(m_frameCount, index, AdvanceBoth, min3,dMN); else - setValue(frameCount, index, ADVANCE_THIS, min2,dMN); + setValue(m_frameCount, index, AdvanceThis, min2,dMN); } } - otherMatcher->last[index]++; + m_otherMatcher->m_last[index]++; } // loop for row (resp. column) - frameCount++; - runCount++; + m_frameCount++; + m_runCount++; - otherMatcher->runCount = 0; - - if (overflow && !silent) - cerr << "WARNING: overflow in distance metric: " - << "frame " << frameCount << ", val = " << mx << endl; - - if (!silent) - std::cerr << "Frame " << frameCount << ", d = " << (mx-mn) << std::endl; + m_otherMatcher->m_runCount = 0; } -int +double Matcher::getValue(int i, int j, bool firstAttempt) { - if (firstPM) - return bestPathCost[i][j - first[i]]; + if (m_firstPM) + return m_bestPathCost[i][j - m_first[i]]; else - return otherMatcher->bestPathCost[j][i - otherMatcher->first[j]]; + return m_otherMatcher->m_bestPathCost[j][i - m_otherMatcher->m_first[j]]; } // getValue() void -Matcher::setValue(int i, int j, int dir, int value, int dMN) +Matcher::setValue(int i, int j, Advance dir, double value, float dMN) { - if (firstPM) { - distance[i][j - first[i]] = (unsigned char)((dMN & MASK) | dir); - bestPathCost[i][j - first[i]] = - (value + (dir==ADVANCE_BOTH? dMN*2: dMN)); + if (m_firstPM) { + + int jdx = j - m_first[i]; + m_distance[i][jdx] = dMN; + m_advance[i][jdx] = dir; + m_bestPathCost[i][jdx] = + (value + (dir == AdvanceBoth ? dMN*2: dMN)); + } else { - if (dir == ADVANCE_THIS) - dir = ADVANCE_OTHER; - else if (dir == ADVANCE_OTHER) - dir = ADVANCE_THIS; - int idx = i - otherMatcher->first[j]; - if (idx == (int)otherMatcher->distYSizes[j]) { + + if (dir == AdvanceThis) { + dir = AdvanceOther; + } else if (dir == AdvanceOther) { + dir = AdvanceThis; + } + + int idx = i - m_otherMatcher->m_first[j]; + + if (idx == (int)m_otherMatcher->m_distYSizes[j]) { // This should never happen, but if we allow arbitrary // pauses in either direction, and arbitrary lengths at // end, it is better than a segmentation fault. std::cerr << "Emergency resize: " << idx << " -> " << idx * 2 << std::endl; - otherMatcher->distYSizes[j] = idx * 2; - otherMatcher->bestPathCost[j] = - (int *)realloc(otherMatcher->bestPathCost[j], - idx * 2 * sizeof(int)); - otherMatcher->distance[j] = - (unsigned char *)realloc(otherMatcher->distance[j], - idx * 2 * sizeof(unsigned char)); + m_otherMatcher->m_distYSizes[j] = idx * 2; + m_otherMatcher->m_bestPathCost[j].resize(idx * 2, 0); + m_otherMatcher->m_distance[j].resize(idx * 2, 0); + m_otherMatcher->m_advance[j].resize(idx * 2, AdvanceNone); } - otherMatcher->distance[j][idx] = (unsigned char)((dMN & MASK) | dir); - otherMatcher->bestPathCost[j][idx] = - (value + (dir==ADVANCE_BOTH? dMN*2: dMN)); + + m_otherMatcher->m_distance[j][idx] = dMN; + m_otherMatcher->m_advance[j][idx] = dir; + m_otherMatcher->m_bestPathCost[j][idx] = + (value + (dir == AdvanceBoth ? dMN*2: dMN)); } } // setValue()
--- a/src/Matcher.h Fri Nov 14 10:24:13 2014 +0000 +++ b/src/Matcher.h Fri Nov 14 10:25:40 2014 +0000 @@ -22,12 +22,8 @@ #include <sstream> #include <cmath> -#define ADVANCE_THIS 1 -#define ADVANCE_OTHER 2 -#define ADVANCE_BOTH 3 -#define MASK 0xfc - #include "DistanceMetric.h" +#include "FeatureExtractor.h" using std::vector; using std::string; @@ -36,92 +32,52 @@ /** Represents an audio stream that can be matched to another audio * stream of the same piece of music. The matching algorithm uses - * dynamic time warping. The distance metric is a Euclidean metric - * on the FFT data with the higher frequencies mapped onto a linear - * scale. + * dynamic time warping. */ class Matcher { public: - enum FrameNormalisation { - - /** Do not normalise audio frames */ - NoFrameNormalisation, - - /** Normalise each frame of audio to have a sum of 1 */ - NormaliseFrameToSum1, - - /** Normalise each frame of audio by the long-term average - * of the summed energy */ - NormaliseFrameToLTAverage, + enum Advance { + AdvanceNone, + AdvanceBoth, + AdvanceThis, + AdvanceOther }; struct Parameters { Parameters(float rate_, double hopTime_, int fftSize_) : sampleRate(rate_), - frameNorm(NormaliseFrameToSum1), distanceNorm(DistanceMetric::NormaliseDistanceToLogSum), - distanceScale(90.0), - useSpectralDifference(true), - useChromaFrequencyMap(false), hopTime(hopTime_), fftSize(fftSize_), blockTime(10.0), - silenceThreshold(0.01), - decay(0.99), maxRunCount(3) {} /** Sample rate of audio */ float sampleRate; - /** Type of audio frame normalisation */ - FrameNormalisation frameNorm; - /** Type of distance metric normalisation */ DistanceMetric::DistanceNormalisation distanceNorm; - /** Scaling factor for distance metric; must guarantee that the - * final value fits in the data type used, that is, unsigned - * char. - */ - double distanceScale; - - /** Flag indicating whether or not the half-wave rectified - * spectral difference should be used in calculating the - * distance metric for pairs of audio frames, instead of the - * straight spectrum values. */ - bool useSpectralDifference; - - /** Flag indicating whether to use a chroma frequency map (12 - * bins) instead of the default warped spectrogram */ - bool useChromaFrequencyMap; - /** Spacing of audio frames (determines the amount of overlap or * skip between frames). This value is expressed in * seconds. */ double hopTime; - + /** Size of an FFT frame in samples. Note that the data passed * in to Matcher is already in the frequency domain, so this * expresses the size of the frame that the caller will be - * providing. - */ + * providing. */ int fftSize; - + /** The width of the search band (error margin) around the current * match position, measured in seconds. Strictly speaking the * width is measured backwards from the current point, since the * algorithm has to work causally. */ double blockTime; - - /** RMS level below which frame is considered silent */ - double silenceThreshold; - - /** Frame-to-frame decay factor in calculating long-term average */ - double decay; /** Maximum number of frames sequentially processed by this * matcher, without a frame of the other matcher being @@ -130,107 +86,6 @@ int maxRunCount; }; -protected: - /** Points to the other performance with which this one is being - * compared. The data for the distance metric and the dynamic - * time warping is shared between the two matchers. In the - * original version, only one of the two performance matchers - * contained the distance metric. (See <code>first</code>) - */ - Matcher *otherMatcher; - - /** Indicates which performance is considered primary (the - * score). This is the performance shown on the vertical axis, - * and referred to as "this" in the codes for the direction of - * DTW steps. */ - bool firstPM; - - /** Configuration parameters */ - Parameters params; - - /** Width of the search band in FFT frames (see <code>blockTime</code>) */ - int blockSize; - - /** The number of frames of audio data which have been read. */ - int frameCount; - - /** Long term average frame energy (in frequency domain - * representation). */ - double ltAverage; - - /** The number of frames sequentially processed by this matcher, - * without a frame of the other matcher being processed. - */ - int runCount; - - /** A mapping function for mapping FFT bins to final frequency - * bins. The mapping is linear (1-1) until the resolution - * reaches 2 points per semitone, then logarithmic with a - * semitone resolution. e.g. for 44.1kHz sampling rate and - * fftSize of 2048 (46ms), bin spacing is 21.5Hz, which is mapped - * linearly for bins 0-34 (0 to 732Hz), and logarithmically for - * the remaining bins (midi notes 79 to 127, bins 35 to 83), - * where all energy above note 127 is mapped into the final - * bin. */ - vector<int> freqMap; - - /** The number of entries in <code>freqMap</code>. */ - int freqMapSize; - - /** The number of values in an externally-supplied feature vector, - * used in preference to freqMap/freqMapSize if constructed with - * the external feature version of the Matcher constructor. If - * this is zero, the internal feature extractor will be used as - * normal. - */ - int externalFeatureSize; - - /** The number of values in the feature vectors actually in - * use. This will be externalFeatureSize if greater than zero, or - * freqMapSize otherwise. - */ - int featureSize; - - /** The most recent frame; used for calculating the frame to frame - * spectral difference. These are therefore frequency warped but - * not yet normalised. */ - vector<double> prevFrame; - vector<double> newFrame; - - /** A block of previously seen frames are stored in this structure - * for calculation of the distance matrix as the new frames are - * read in. One can think of the structure of the array as a - * circular buffer of vectors. These are the frames with all - * applicable processing applied (e.g. spectral difference, - * normalisation), unlike prevFrame and newFrame. The total - * energy of frames[i] is stored in totalEnergies[i]. */ - vector<vector<double> > frames; - - /** The total energy of each frame in the frames block. */ - vector<double> totalEnergies; - - /** The best path cost matrix. */ - int **bestPathCost; - - /** The distance matrix. */ - unsigned char **distance; - - /** The bounds of each row of data in the distance and path cost matrices.*/ - int *first; - int *last; - - /** Height of each column in distance and bestPathCost matrices */ - int *distYSizes; - - /** Width of distance and bestPathCost matrices and first and last vectors */ - int distXSize; - - bool initialised; - - /** Disable or enable debugging output */ - static bool silent; - -public: /** Constructor for Matcher. * * @param p The Matcher representing the performance with which @@ -238,7 +93,9 @@ * between the two matchers (currently one possesses the distance * matrix and optimal path matrix). */ - Matcher(Parameters parameters, Matcher *p); + Matcher(Parameters parameters, + FeatureExtractor::Parameters featureParams, + Matcher *p); /** Constructor for Matcher using externally supplied features. * A Matcher made using this constructor will not carry out its @@ -257,76 +114,31 @@ ~Matcher(); - /** For debugging, outputs information about the Matcher to - * standard error. - */ - void print(); - /** Adds a link to the Matcher object representing the performance * which is going to be matched to this one. * * @param p the Matcher representing the other performance */ void setOtherMatcher(Matcher *p) { - otherMatcher = p; + m_otherMatcher = p; } // setOtherMatcher() int getFrameCount() { - return frameCount; + return m_frameCount; } - /** - * Return the feature vector size that will be used for the given - * parameters. - */ - static int getFeatureSizeFor(Parameters params); - protected: - template <typename T> - void initVector(vector<T> &vec, int sz, T dflt = 0) { - vec.clear(); - while ((int)vec.size() < sz) vec.push_back(dflt); - } - - template <typename T> - void initMatrix(vector<vector<T> > &mat, int hsz, int vsz, - T dflt = 0, int fillTo = -1) { - mat.clear(); - if (fillTo < 0) fillTo = hsz; - for (int i = 0; i < hsz; ++i) { - mat.push_back(vector<T>()); - if (i < fillTo) { - while ((int)mat[i].size() < vsz) { - mat[i].push_back(dflt); - } - } - } - } - + /** Create internal structures and reset. */ void init(); - void makeFreqMap(); + /** The distXSize value has changed: resize internal buffers. */ + void size(); - /** Creates a map of FFT frequency bins to comparison bins. Where - * the spacing of FFT bins is less than 0.5 semitones, the - * mapping is one to one. Where the spacing is greater than 0.5 - * semitones, the FFT energy is mapped into semitone-wide - * bins. No scaling is performed; that is the energy is summed - * into the comparison bins. See also consumeFrame() - */ - void makeStandardFrequencyMap(); - - void makeChromaFrequencyMap(); - - /** Processes a frame of audio data by first computing the STFT - * with a Hamming window, then mapping the frequency bins into a - * part-linear part-logarithmic array, then (optionally) - * computing the half-wave rectified spectral difference from the - * previous frame, then (optionally) normalising to a sum of 1, - * then calculating the distance to all frames stored in the - * otherMatcher and storing them in the distance matrix, and - * finally updating the optimal path matrix using the dynamic - * time warping algorithm. + /** Process a frequency-domain frame of audio data using the + * built-in FeatureExtractor, then calculating the distance to + * all frames stored in the otherMatcher and storing them in the + * distance matrix, and finally updating the optimal path matrix + * using the dynamic time warping algorithm. * * Return value is the frame (post-processed, with warping, * rectification, and normalisation as appropriate). @@ -356,7 +168,7 @@ * @param j the frame number of the other Matcher * @return the cost of the minimum cost path to this location */ - int getValue(int i, int j, bool firstAttempt); + double getValue(int i, int j, bool firstAttempt); /** Stores entries in the distance matrix and the optimal path matrix. * @@ -367,12 +179,73 @@ * @param value the cost of the minimum path except the current step * @param dMN the distance cost between the two frames */ - void setValue(int i, int j, int dir, int value, int dMN); + void setValue(int i, int j, Advance dir, double value, float dMN); - vector<double> processFrameFromFreqData(double *, double *); void calcAdvance(); - DistanceMetric metric; + /** Points to the other performance with which this one is being + * compared. The data for the distance metric and the dynamic + * time warping is shared between the two matchers. In the + * original version, only one of the two performance matchers + * contained the distance metric. (See <code>first</code>) + */ + Matcher *m_otherMatcher; + + /** Indicates which performance is considered primary (the + * score). This is the performance shown on the vertical axis, + * and referred to as "this" in the codes for the direction of + * DTW steps. */ + bool m_firstPM; + + /** Configuration parameters */ + Parameters m_params; + + /** Width of the search band in FFT frames (see <code>blockTime</code>) */ + int m_blockSize; + + /** The number of frames of audio data which have been read. */ + int m_frameCount; + + /** The number of frames sequentially processed by this matcher, + * without a frame of the other matcher being processed. + */ + int m_runCount; + + /** The number of values in a feature vector. */ + int m_featureSize; + + /** A block of previously seen feature frames is stored in this + * structure for calculation of the distance matrix as the new + * frames are received. One can think of the structure of the + * array as a circular buffer of vectors. */ + vector<vector<double> > m_frames; + + /** The best path cost matrix. */ + vector<vector<double> > m_bestPathCost; + + /** The distance matrix. */ + vector<vector<float> > m_distance; + + /** The advance direction matrix. */ + vector<vector<Advance> > m_advance; + + /** The bounds of each row of data in the distance, path cost, and + * advance direction matrices.*/ + vector<int> m_first; + vector<int> m_last; + + /** Height of each column in distance, path cost, and advance + * direction matrices. */ + vector<int> m_distYSizes; + + /** Width of distance, path cost, and advance direction matrices + * and first and last vectors */ + int m_distXSize; + + bool m_initialised; + + FeatureExtractor m_featureExtractor; + DistanceMetric m_metric; friend class MatchFeeder; friend class MatchFeatureFeeder;