cannam@0: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ cannam@0: cannam@0: /* cannam@0: Vamp feature extraction plugin using the MATCH audio alignment cannam@0: algorithm. cannam@0: cannam@0: Centre for Digital Music, Queen Mary, University of London. Chris@236: Copyright (c) 2007-2020 Simon Dixon, Chris Cannam, and Queen Mary Chris@230: University of London, Copyright (c) 2014-2015 Tido GmbH. cannam@0: cannam@0: This program is free software; you can redistribute it and/or cannam@0: modify it under the terms of the GNU General Public License as cannam@0: published by the Free Software Foundation; either version 2 of the cannam@0: License, or (at your option) any later version. See the file cannam@0: COPYING included with this distribution for more information. cannam@0: */ cannam@0: cannam@0: #include "Matcher.h" cannam@0: cannam@0: #include cannam@0: cannam@4: #include Chris@16: #include cannam@4: Chris@72: using namespace std; Chris@72: Chris@10: //#define DEBUG_MATCHER 1 Chris@220: //#define PERFORM_ERROR_CHECKS 1 Chris@10: Chris@143: Matcher::Matcher(Parameters parameters, DistanceMetric::Parameters dparams, Chris@143: Matcher *p) : Chris@43: m_params(parameters), Chris@143: m_metric(dparams) Chris@23: { Chris@23: #ifdef DEBUG_MATCHER Chris@143: cerr << "*** Matcher: hopTime = " << parameters.hopTime Chris@140: << ", blockTime = " << parameters.blockTime Chris@140: << ", maxRunCount = " << parameters.maxRunCount Chris@140: << ", diagonalWeight = " << parameters.diagonalWeight << endl; Chris@23: #endif Chris@140: Chris@43: m_otherMatcher = p; // the first matcher will need this to be set later Chris@43: m_firstPM = (!p); Chris@43: m_frameCount = 0; Chris@43: m_runCount = 0; Chris@43: m_blockSize = 0; Chris@202: m_distXSize = 0; cannam@0: Chris@180: m_blockSize = int(m_params.blockTime / m_params.hopTime + 0.5); Chris@15: #ifdef DEBUG_MATCHER Chris@43: cerr << "Matcher: m_blockSize = " << m_blockSize << endl; Chris@15: #endif cannam@0: Chris@43: m_initialised = false; Chris@23: } cannam@0: cannam@0: Matcher::~Matcher() cannam@0: { Chris@10: #ifdef DEBUG_MATCHER Chris@15: cerr << "Matcher(" << this << ")::~Matcher()" << endl; Chris@10: #endif cannam@0: } cannam@0: cannam@0: void cannam@0: Matcher::init() cannam@0: { Chris@43: if (m_initialised) return; cannam@0: Chris@182: m_features = featureseq_t(m_blockSize); cannam@0: Chris@43: m_distXSize = m_blockSize * 2; Chris@45: Chris@41: size(); cannam@0: Chris@43: m_frameCount = 0; Chris@43: m_runCount = 0; Chris@38: Chris@43: m_initialised = true; Chris@16: } Chris@16: Chris@87: bool Chris@203: Matcher::isAvailable(int i, int j) Chris@154: { Chris@203: if (m_firstPM) { Chris@203: if (isInRange(i, j)) { Chris@214: return (m_distance[i][j - m_first[i]] != INVALID_DISTANCE); Chris@203: } else { Chris@203: return false; Chris@154: } Chris@203: } else { Chris@203: return m_otherMatcher->isAvailable(j, i); Chris@154: } Chris@154: } Chris@154: Chris@154: bool Chris@203: Matcher::isRowAvailable(int i) Chris@154: { Chris@203: if (m_firstPM) { Chris@203: Chris@203: if (i < 0 || i >= int(m_first.size())) return false; Chris@214: for (auto c: m_distance[i]) { Chris@214: if (c != INVALID_DISTANCE) return true; Chris@203: } Chris@203: return false; Chris@203: Chris@203: } else { Chris@203: return m_otherMatcher->isColAvailable(i); Chris@203: } Chris@203: } Chris@203: Chris@203: bool Chris@203: Matcher::isColAvailable(int j) Chris@203: { Chris@203: if (m_firstPM) { Chris@203: for (int i = 0; i < int(m_first.size()); ++i) { Chris@208: if (j >= m_first[i] && j < m_last[i]) { Chris@214: if (m_distance[i][j - m_first[i]] != INVALID_DISTANCE) { Chris@203: return true; Chris@203: } Chris@203: } Chris@203: } Chris@203: return false; Chris@203: } else { Chris@203: return m_otherMatcher->isRowAvailable(j); Chris@203: } Chris@154: } Chris@154: Chris@154: bool Chris@87: Matcher::isInRange(int i, int j) Chris@87: { Chris@87: if (m_firstPM) { Chris@87: return ((i >= 0) && Chris@87: (i < int(m_first.size())) && Chris@87: (j >= m_first[i]) && Chris@214: (j < int(m_first[i] + m_distance[i].size()))); Chris@87: } else { Chris@87: return m_otherMatcher->isInRange(j, i); Chris@87: } Chris@87: } Chris@87: Chris@87: pair Chris@205: Matcher::getColRangeForRow(int i) Chris@87: { Chris@204: if (m_firstPM) { Chris@208: #ifdef PERFORM_ERROR_CHECKS Chris@204: if (i < 0 || i >= int(m_first.size())) { Chris@206: cerr << "ERROR: Matcher::getColRangeForRow(" << i << "): Index out of range" Chris@204: << endl; Chris@204: throw "Index out of range"; Chris@204: } Chris@208: #endif Chris@208: return pair(m_first[i], m_last[i]); Chris@87: } else { Chris@205: return m_otherMatcher->getRowRangeForCol(i); Chris@87: } Chris@87: } Chris@87: Chris@87: pair Chris@206: Matcher::getRowRangeForCol(int i) Chris@87: { Chris@204: if (m_firstPM) { Chris@208: #ifdef PERFORM_ERROR_CHECKS Chris@206: if (i < 0 || i >= int(m_otherMatcher->m_first.size())) { Chris@206: cerr << "ERROR: Matcher::getRowRangeForCol(" << i << "): Index out of range" Chris@204: << endl; Chris@204: throw "Index out of range"; Chris@204: } Chris@208: #endif Chris@208: return pair(m_otherMatcher->m_first[i], Chris@208: m_otherMatcher->m_last[i]); Chris@204: } else { Chris@206: return m_otherMatcher->getColRangeForRow(i); Chris@204: } Chris@87: } Chris@87: Chris@182: distance_t Chris@87: Matcher::getDistance(int i, int j) Chris@87: { Chris@87: if (m_firstPM) { Chris@208: #ifdef PERFORM_ERROR_CHECKS Chris@87: if (!isInRange(i, j)) { Chris@87: cerr << "ERROR: Matcher::getDistance(" << i << ", " << j << "): " Chris@87: << "Location is not in range" << endl; Chris@87: throw "Distance not available"; Chris@87: } Chris@208: #endif Chris@182: distance_t dist = m_distance[i][j - m_first[i]]; Chris@208: #ifdef PERFORM_ERROR_CHECKS Chris@212: if (dist == INVALID_DISTANCE) { Chris@87: cerr << "ERROR: Matcher::getDistance(" << i << ", " << j << "): " Chris@87: << "Location is in range, but distance (" Chris@191: << distance_print_t(dist) Chris@191: << ") is invalid or has not been set" << endl; Chris@87: throw "Distance not available"; Chris@87: } Chris@208: #endif Chris@87: return dist; Chris@87: } else { Chris@87: return m_otherMatcher->getDistance(j, i); Chris@87: } Chris@87: } Chris@87: Chris@87: void Chris@182: Matcher::setDistance(int i, int j, distance_t distance) Chris@87: { Chris@87: if (m_firstPM) { Chris@208: #ifdef PERFORM_ERROR_CHECKS Chris@87: if (!isInRange(i, j)) { Chris@87: cerr << "ERROR: Matcher::setDistance(" << i << ", " << j << ", " Chris@191: << distance_print_t(distance) Chris@191: << "): Location is out of range" << endl; Chris@87: throw "Indices out of range"; Chris@87: } Chris@208: #endif Chris@87: m_distance[i][j - m_first[i]] = distance; Chris@87: } else { Chris@87: m_otherMatcher->setDistance(j, i, distance); Chris@87: } Chris@87: } Chris@87: Chris@191: normpathcost_t Chris@135: Matcher::getNormalisedPathCost(int i, int j) Chris@135: { Chris@135: // normalised for path length. 1+ prevents division by zero here Chris@191: return normpathcost_t(getPathCost(i, j)) / normpathcost_t(1 + i + j); Chris@135: } Chris@135: Chris@182: pathcost_t Chris@87: Matcher::getPathCost(int i, int j) Chris@87: { Chris@87: if (m_firstPM) { Chris@203: #ifdef PERFORM_ERROR_CHECKS Chris@87: if (!isAvailable(i, j)) { Chris@87: if (!isInRange(i, j)) { Chris@87: cerr << "ERROR: Matcher::getPathCost(" << i << ", " << j << "): " Chris@87: << "Location is not in range" << endl; Chris@87: } else { Chris@87: cerr << "ERROR: Matcher::getPathCost(" << i << ", " << j << "): " Chris@87: << "Location is in range, but pathCost (" Chris@87: << m_bestPathCost[i][j - m_first[i]] Chris@87: << ") is invalid or has not been set" << endl; Chris@87: } Chris@87: throw "Path cost not available"; Chris@87: } Chris@203: #endif Chris@87: return m_bestPathCost[i][j - m_first[i]]; Chris@87: } else { Chris@87: return m_otherMatcher->getPathCost(j, i); Chris@87: } Chris@87: } Chris@87: Chris@87: void Chris@182: Matcher::setPathCost(int i, int j, advance_t dir, pathcost_t pathCost) Chris@87: { Chris@87: if (m_firstPM) { Chris@208: #ifdef PERFORM_ERROR_CHECKS Chris@87: if (!isInRange(i, j)) { Chris@87: cerr << "ERROR: Matcher::setPathCost(" << i << ", " << j << ", " Chris@87: << dir << ", " << pathCost Chris@87: << "): Location is out of range" << endl; Chris@87: throw "Indices out of range"; Chris@87: } Chris@208: #endif Chris@87: m_advance[i][j - m_first[i]] = dir; Chris@87: m_bestPathCost[i][j - m_first[i]] = pathCost; Chris@87: } else { Chris@87: if (dir == AdvanceThis) { Chris@87: dir = AdvanceOther; Chris@87: } else if (dir == AdvanceOther) { Chris@87: dir = AdvanceThis; Chris@87: } Chris@87: m_otherMatcher->setPathCost(j, i, dir, pathCost); Chris@87: } Chris@87: } Chris@87: cannam@0: void Chris@41: Matcher::size() cannam@0: { Chris@43: m_first.resize(m_distXSize, 0); Chris@43: m_last.resize(m_distXSize, 0); Chris@206: Chris@206: if (m_firstPM) { Chris@206: int distSize = (m_params.maxRunCount + 1) * m_blockSize; Chris@212: m_bestPathCost.resize(m_distXSize, pathcostvec_t(distSize, INVALID_PATHCOST)); Chris@212: m_distance.resize(m_distXSize, distancevec_t(distSize, INVALID_DISTANCE)); Chris@206: m_advance.resize(m_distXSize, advancevec_t(distSize, AdvanceNone)); Chris@206: } Chris@38: } cannam@0: Chris@23: void Chris@182: Matcher::consumeFeatureVector(const feature_t &feature) Chris@23: { Chris@43: if (!m_initialised) init(); Chris@43: int frameIndex = m_frameCount % m_blockSize; Chris@182: m_features[frameIndex] = feature; Chris@23: calcAdvance(); Chris@21: } Chris@21: Chris@211: pathcost_t Chris@211: Matcher::addToCost(pathcost_t cost, pathcost_t increment) Chris@211: { Chris@212: if (PATHCOST_MAX - increment < cost) { Chris@212: return PATHCOST_MAX; Chris@211: } else { Chris@211: return cost + pathcost_t(increment); Chris@211: } Chris@211: } Chris@211: Chris@21: void Chris@21: Matcher::calcAdvance() Chris@21: { Chris@43: int frameIndex = m_frameCount % m_blockSize; Chris@21: Chris@43: if (m_frameCount >= m_distXSize) { Chris@211: m_distXSize = int(m_distXSize * 1.2); Chris@41: size(); cannam@0: } Chris@207: Chris@43: if (m_firstPM && (m_frameCount >= m_blockSize)) { Chris@207: // Memory reduction for old rows Chris@207: int oldidx = m_frameCount - m_blockSize; Chris@207: int len = m_last[oldidx] - m_first[oldidx]; Chris@207: m_distance[oldidx].resize(len); Chris@209: m_distance[oldidx].shrink_to_fit(); Chris@207: m_bestPathCost[oldidx].resize(len); Chris@209: m_bestPathCost[oldidx].shrink_to_fit(); Chris@207: m_advance[oldidx].resize(len); Chris@209: m_advance[oldidx].shrink_to_fit(); cannam@0: } cannam@0: Chris@43: int stop = m_otherMatcher->m_frameCount; Chris@43: int index = stop - m_blockSize; Chris@86: if (index < 0) index = 0; Chris@86: Chris@43: m_first[m_frameCount] = index; Chris@43: m_last[m_frameCount] = stop; cannam@0: cannam@0: for ( ; index < stop; index++) { Chris@26: Chris@188: distance_t distance = m_metric.calcDistance Chris@182: (m_features[frameIndex], Chris@182: m_otherMatcher->m_features[index % m_blockSize]); Chris@26: Chris@188: pathcost_t straightIncrement(distance); Chris@188: pathcost_t diagIncrement = pathcost_t(distance * m_params.diagonalWeight); Chris@89: Chris@86: if ((m_frameCount == 0) && (index == 0)) { // first element Chris@86: Chris@86: updateValue(0, 0, AdvanceNone, Chris@86: 0, Chris@86: distance); Chris@86: Chris@86: } else if (m_frameCount == 0) { // first row Chris@86: Chris@220: pathcost_t cost {}; Chris@220: Chris@220: if (isInRange(0, index - 1)) { Chris@220: cost = getPathCost(0, index - 1); Chris@220: } Chris@220: Chris@220: updateValue(0, index, AdvanceOther, cost, distance); Chris@86: Chris@86: } else if (index == 0) { // first column Chris@86: Chris@220: pathcost_t cost {}; Chris@220: Chris@220: if (isInRange(m_frameCount - 1, 0)) { Chris@220: cost = getPathCost(m_frameCount - 1, 0); Chris@220: } Chris@220: Chris@220: updateValue(m_frameCount, index, AdvanceThis, cost, distance); Chris@86: Chris@86: } else if (index == m_otherMatcher->m_frameCount - m_blockSize) { Chris@86: cannam@0: // missing value(s) due to cutoff cannam@0: // - no previous value in current row (resp. column) cannam@0: // - no diagonal value if prev. dir. == curr. dirn Chris@86: Chris@182: pathcost_t min2 = getPathCost(m_frameCount - 1, index); Chris@86: Chris@91: // cerr << "NOTE: missing value at i = " << m_frameCount << ", j = " Chris@91: // << index << " (first = " << m_firstPM << ")" << endl; Chris@86: Chris@43: // if ((m_firstPM && (first[m_frameCount - 1] == index)) || Chris@43: // (!m_firstPM && (m_last[index-1] < m_frameCount))) Chris@86: if (m_first[m_frameCount - 1] == index) { Chris@86: Chris@86: updateValue(m_frameCount, index, AdvanceThis, Chris@86: min2, distance); Chris@86: Chris@86: } else { Chris@86: Chris@182: pathcost_t min1 = getPathCost(m_frameCount - 1, index - 1); Chris@211: if (addToCost(min1, diagIncrement) <= Chris@211: addToCost(min2, straightIncrement)) { Chris@86: updateValue(m_frameCount, index, AdvanceBoth, Chris@86: min1, distance); Chris@86: } else { Chris@86: updateValue(m_frameCount, index, AdvanceThis, Chris@86: min2, distance); Chris@86: } cannam@0: } Chris@86: cannam@0: } else { Chris@86: Chris@182: pathcost_t min1 = getPathCost(m_frameCount, index - 1); Chris@182: pathcost_t min2 = getPathCost(m_frameCount - 1, index); Chris@182: pathcost_t min3 = getPathCost(m_frameCount - 1, index - 1); Chris@87: Chris@211: pathcost_t cost1 = addToCost(min1, straightIncrement); Chris@211: pathcost_t cost2 = addToCost(min2, straightIncrement); Chris@211: pathcost_t cost3 = addToCost(min3, diagIncrement); Chris@93: Chris@93: // Choosing is easy if there is a strict cheapest of the Chris@93: // three. If two or more share the lowest cost, we choose Chris@93: // in order of preference: cost3 (AdvanceBoth), cost2 Chris@94: // (AdvanceThis), cost1 (AdvanceOther) if we are the first Chris@94: // matcher; and cost3 (AdvanceBoth), cost1 (AdvanceOther), Chris@94: // cost2 (AdvanceThis) if we are the second matcher. That Chris@94: // is, we always prioritise the diagonal followed by the Chris@94: // first matcher. Chris@94: Chris@94: if (( m_firstPM && (cost1 < cost2)) || Chris@94: (!m_firstPM && (cost1 <= cost2))) { Chris@89: if (cost3 <= cost1) { Chris@86: updateValue(m_frameCount, index, AdvanceBoth, Chris@86: min3, distance); Chris@86: } else { Chris@86: updateValue(m_frameCount, index, AdvanceOther, Chris@86: min1, distance); Chris@86: } cannam@0: } else { Chris@89: if (cost3 <= cost2) { Chris@86: updateValue(m_frameCount, index, AdvanceBoth, Chris@86: min3, distance); Chris@86: } else { Chris@86: updateValue(m_frameCount, index, AdvanceThis, Chris@86: min2, distance); Chris@86: } cannam@0: } cannam@0: } Chris@86: Chris@43: m_otherMatcher->m_last[index]++; cannam@0: } // loop for row (resp. column) cannam@0: Chris@43: m_frameCount++; Chris@43: m_runCount++; cannam@0: Chris@43: m_otherMatcher->m_runCount = 0; Chris@21: } cannam@0: cannam@0: void Chris@182: Matcher::updateValue(int i, int j, advance_t dir, pathcost_t value, distance_t distance) cannam@0: { Chris@188: pathcost_t increment = distance; Chris@83: if (dir == AdvanceBoth) { Chris@188: increment = pathcost_t(increment * m_params.diagonalWeight); Chris@188: } Chris@188: Chris@211: pathcost_t newValue = addToCost(value, increment); Chris@212: if (newValue == PATHCOST_MAX) { Chris@188: cerr << "ERROR: Path cost overflow at i=" << i << ", j=" << j << ": " Chris@212: << value << " + " << increment << " >= " << PATHCOST_MAX << endl; Chris@212: newValue = PATHCOST_MAX; Chris@83: } Chris@83: Chris@43: if (m_firstPM) { Chris@45: Chris@96: setDistance(i, j, distance); Chris@188: setPathCost(i, j, dir, newValue); Chris@45: cannam@0: } else { Chris@45: Chris@86: if (dir == AdvanceThis) dir = AdvanceOther; Chris@86: else if (dir == AdvanceOther) dir = AdvanceThis; Chris@84: Chris@43: int idx = i - m_otherMatcher->m_first[j]; Chris@45: Chris@188: if (idx < 0 || size_t(idx) == m_otherMatcher->m_distance[j].size()) { cannam@0: // This should never happen, but if we allow arbitrary cannam@0: // pauses in either direction, and arbitrary lengths at cannam@0: // end, it is better than a segmentation fault. Chris@217: // cerr << "Emergency resize: " << idx << " -> " << idx * 2 << endl; Chris@212: m_otherMatcher->m_bestPathCost[j].resize(idx * 2, INVALID_PATHCOST); Chris@212: m_otherMatcher->m_distance[j].resize(idx * 2, INVALID_DISTANCE); Chris@46: m_otherMatcher->m_advance[j].resize(idx * 2, AdvanceNone); cannam@0: } Chris@45: Chris@96: m_otherMatcher->setDistance(j, i, distance); Chris@188: m_otherMatcher->setPathCost(j, i, dir, newValue); cannam@0: } Chris@71: } cannam@0: Chris@181: advance_t Chris@72: Matcher::getAdvance(int i, int j) Chris@72: { Chris@72: if (m_firstPM) { Chris@208: #ifdef PERFORM_ERROR_CHECKS Chris@72: if (!isInRange(i, j)) { Chris@72: cerr << "ERROR: Matcher::getAdvance(" << i << ", " << j << "): " Chris@72: << "Location is not in range" << endl; Chris@72: throw "Advance not available"; Chris@72: } Chris@208: #endif Chris@72: return m_advance[i][j - m_first[i]]; Chris@72: } else { Chris@72: return m_otherMatcher->getAdvance(j, i); Chris@72: } Chris@72: } Chris@202: Chris@202: static double k(size_t sz) Chris@202: { Chris@202: return double(sz) / 1024.0; Chris@202: } Chris@202: Chris@232: Matcher::MemoryStats Chris@232: Matcher::getMemoryStats() const Chris@232: { Chris@232: MemoryStats stats; Chris@233: stats.features_k = 0.0; Chris@233: if (!m_features.empty()) { Chris@233: stats.features_k = Chris@233: k(m_features.size() * m_features[0].size() * sizeof(featurebin_t)); Chris@233: } Chris@232: Chris@232: size_t cells = 0; Chris@232: for (const auto &d: m_distance) { Chris@232: cells += d.size(); Chris@232: } Chris@232: Chris@232: stats.pathcosts_k = k(cells * sizeof(pathcost_t)); Chris@232: stats.distances_k = k(cells * sizeof(distance_t)); Chris@232: stats.advances_k = k(cells * sizeof(advance_t)); Chris@232: Chris@232: if (m_firstPM && m_otherMatcher) { Chris@232: stats = stats + m_otherMatcher->getMemoryStats(); Chris@232: } Chris@232: Chris@232: return stats; Chris@232: } Chris@232: Chris@202: void Chris@202: Matcher::printStats() Chris@202: { Chris@202: if (m_firstPM) cerr << endl; Chris@202: Chris@202: cerr << "Matcher[" << this << "] (" << (m_firstPM ? "first" : "second") << "):" << endl; Chris@202: cerr << "- block size " << m_blockSize << ", frame count " << m_frameCount << ", dist x size " << m_distXSize << ", initialised " << m_initialised << endl; Chris@202: Chris@202: if (m_features.empty()) { Chris@202: cerr << "- have no features yet" << endl; Chris@202: } else { Chris@232: cerr << "- have " << m_features.size() << " features of " << m_features[0].size() << " bins each" << endl; Chris@202: } Chris@202: Chris@202: size_t cells = 0; Chris@202: for (const auto &d: m_distance) { Chris@202: cells += d.size(); Chris@202: } Chris@202: if (m_distance.empty()) { Chris@202: cerr << "- have no cells in matrix" << endl; Chris@202: } else { Chris@202: cerr << "- have " << m_distance.size() << " cols in matrix with avg " Chris@203: << double(cells) / double(m_distance.size()) << " rows, total " Chris@202: << cells << " cells" << endl; Chris@202: } Chris@202: Chris@202: if (m_firstPM && m_otherMatcher) { Chris@202: m_otherMatcher->printStats(); Chris@232: MemoryStats stats = getMemoryStats(); Chris@232: cerr << "Memory: " Chris@232: << "features " << stats.features_k << "K, " Chris@232: << "path costs " << stats.pathcosts_k << "K, " Chris@232: << "distances " << stats.distances_k << "K,\n " Chris@232: << "advances " << stats.advances_k << "K, " Chris@232: << "total " << stats.total_k() << "K" Chris@232: << endl; Chris@202: cerr << endl; Chris@202: } Chris@202: } Chris@202: