cannam@0: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
cannam@0: 
cannam@0: /*
cannam@0:     Vamp feature extraction plugin using the MATCH audio alignment
cannam@0:     algorithm.
cannam@0: 
cannam@0:     Centre for Digital Music, Queen Mary, University of London.
cannam@0:     This file copyright 2007 Simon Dixon, Chris Cannam and QMUL.
cannam@0:     
cannam@0:     This program is free software; you can redistribute it and/or
cannam@0:     modify it under the terms of the GNU General Public License as
cannam@0:     published by the Free Software Foundation; either version 2 of the
cannam@0:     License, or (at your option) any later version.  See the file
cannam@0:     COPYING included with this distribution for more information.
cannam@0: */
cannam@0: 
cannam@0: #include "MatchVampPlugin.h"
cannam@0: 
cannam@0: #include "Matcher.h"
cannam@0: #include "MatchFeeder.h"
cannam@0: #include "Path.h"
cannam@0: 
cannam@0: #include <vamp/vamp.h>
cannam@0: #include <vamp-sdk/PluginAdapter.h>
cannam@0: #include <vamp-sdk/RealTime.h>
cannam@0: 
cannam@0: #include <vector>
cannam@0: #include <algorithm>
cannam@0: 
cannam@0: //static int extant = 0;
cannam@0: 
cannam@0: #ifdef _WIN32
cannam@0: HANDLE
cannam@0: MatchVampPlugin::m_serialisingMutex;
cannam@0: #else
cannam@0: pthread_mutex_t 
cannam@0: MatchVampPlugin::m_serialisingMutex;
cannam@0: #endif
cannam@0: 
cannam@0: bool
cannam@0: MatchVampPlugin::m_serialisingMutexInitialised = false;
cannam@0: 
cannam@0: MatchVampPlugin::MatchVampPlugin(float inputSampleRate) :
cannam@0:     Plugin(inputSampleRate),
cannam@6:     m_stepSize(0),
cannam@0:     m_serialise(false),
cannam@0:     m_begin(true),
cannam@0:     m_locked(false)
cannam@0: {
cannam@0:     if (!m_serialisingMutexInitialised) {
cannam@0:         m_serialisingMutexInitialised = true;
cannam@0: #ifdef _WIN32
cannam@0:         m_serialisingMutex = CreateMutex(NULL, FALSE, NULL);
cannam@0: #else
cannam@0:         pthread_mutex_init(&m_serialisingMutex, 0);
cannam@0: #endif
cannam@0:     }
cannam@0: 
cannam@0:     pm1 = 0;
cannam@0:     pm2 = 0;
cannam@0:     feeder = 0;
cannam@0: //    std::cerr << "MatchVampPlugin::MatchVampPlugin(" << this << "): extant = " << ++extant << std::endl;
cannam@0: }
cannam@0: 
cannam@0: MatchVampPlugin::~MatchVampPlugin()
cannam@0: {
cannam@0: //    std::cerr << "MatchVampPlugin::~MatchVampPlugin(" << this << "): extant = " << --extant << std::endl;
cannam@0: 
cannam@0:     delete feeder;
cannam@0:     delete pm1;
cannam@0:     delete pm2;
cannam@0: 
cannam@0:     if (m_locked) {
cannam@0: #ifdef _WIN32
cannam@0:         ReleaseMutex(m_serialisingMutex);
cannam@0: #else
cannam@0:         pthread_mutex_unlock(&m_serialisingMutex);
cannam@0: #endif
cannam@0:         m_locked = false;
cannam@0:     }
cannam@0: }
cannam@0: 
cannam@0: string
cannam@0: MatchVampPlugin::getIdentifier() const
cannam@0: {
cannam@0:     return "match";
cannam@0: }
cannam@0: 
cannam@0: string
cannam@0: MatchVampPlugin::getName() const
cannam@0: {
cannam@0:     return "Match Performance Aligner";
cannam@0: }
cannam@0: 
cannam@0: string
cannam@0: MatchVampPlugin::getDescription() const
cannam@0: {
cannam@0:     return "Calculate alignment between two performances in separate channel inputs";
cannam@0: }
cannam@0: 
cannam@0: string
cannam@0: MatchVampPlugin::getMaker() const
cannam@0: {
cannam@0:     return "Simon Dixon (plugin by Chris Cannam)";
cannam@0: }
cannam@0: 
cannam@0: int
cannam@0: MatchVampPlugin::getPluginVersion() const
cannam@0: {
cannam@0:     return 1;
cannam@0: }
cannam@0: 
cannam@0: string
cannam@0: MatchVampPlugin::getCopyright() const
cannam@0: {
cannam@0:     return "GPL";
cannam@0: }
cannam@0: 
cannam@0: MatchVampPlugin::ParameterList
cannam@0: MatchVampPlugin::getParameterDescriptors() const
cannam@0: {
cannam@0:     ParameterList list;
cannam@0: 
cannam@0:     ParameterDescriptor desc;
cannam@0:     desc.identifier = "serialise";
cannam@0:     desc.name = "Serialise Plugin Invocations";
cannam@0:     desc.description = "Reduce potential memory load at the expense of multiprocessor performance by serialising multi-threaded plugin runs";
cannam@0:     desc.minValue = 0;
cannam@0:     desc.maxValue = 1;
cannam@0:     desc.defaultValue = 0;
cannam@0:     desc.isQuantized = true;
cannam@0:     desc.quantizeStep = 1;
cannam@0:     list.push_back(desc);
cannam@0: 
cannam@0:     return list;
cannam@0: }
cannam@0: 
cannam@0: float
cannam@0: MatchVampPlugin::getParameter(std::string name) const
cannam@0: {
cannam@0:     if (name == "serialise") {
cannam@0:         return m_serialise ? 1.0 : 0.0; 
cannam@0:     }
cannam@0:     return 0.0;
cannam@0: }
cannam@0: 
cannam@0: void
cannam@0: MatchVampPlugin::setParameter(std::string name, float value)
cannam@0: {
cannam@0:     if (name == "serialise") {
cannam@0:         m_serialise = (value > 0.5);
cannam@0:         std::cerr << "MatchVampPlugin::setParameter: set serialise to " << m_serialise << std::endl;
cannam@0:     }
cannam@0: }
cannam@0: 
cannam@0: size_t
cannam@0: MatchVampPlugin::getPreferredStepSize() const
cannam@0: {
cannam@0:     if (!pm1) createMatchers();
cannam@0:     return pm1->getHopSize();
cannam@0: }
cannam@0: 
cannam@0: size_t
cannam@0: MatchVampPlugin::getPreferredBlockSize() const
cannam@0: {
cannam@0:     if (!pm1) createMatchers();
cannam@0:     return pm1->getFFTSize();
cannam@0: }
cannam@0: 
cannam@0: void
cannam@0: MatchVampPlugin::createMatchers() const
cannam@0: {
cannam@0:     pm1 = new Matcher(m_inputSampleRate, 0);
cannam@0:     pm2 = new Matcher(m_inputSampleRate, pm1);
cannam@0:     pm1->setOtherMatcher(pm2);
cannam@0:     feeder = new MatchFeeder(pm1, pm2);
cannam@0: }
cannam@0: 
cannam@0: bool
cannam@0: MatchVampPlugin::initialise(size_t channels, size_t stepSize, size_t blockSize)
cannam@0: {
cannam@0:     if (!pm1) createMatchers();
cannam@0:     if (channels < getMinChannelCount() ||
cannam@0: 	channels > getMaxChannelCount()) return false;
cannam@1:     if (stepSize > blockSize/2 ||
cannam@0:         blockSize != getPreferredBlockSize()) return false;
cannam@6:     m_stepSize = stepSize;
cannam@1:     pm1->setHopSize(stepSize);
cannam@1:     pm2->setHopSize(stepSize);
cannam@0:     m_begin = true;
cannam@0:     m_locked = false;
cannam@0:     return true;
cannam@0: }
cannam@0: 
cannam@0: void
cannam@0: MatchVampPlugin::reset()
cannam@0: {
cannam@6:     delete feeder;
cannam@6:     delete pm1;
cannam@6:     delete pm2;
cannam@6:     feeder = 0;
cannam@6:     pm1 = 0;
cannam@6:     pm2 = 0;
cannam@6: 
cannam@6:     createMatchers();
cannam@6:     pm1->setHopSize(m_stepSize);
cannam@6:     pm2->setHopSize(m_stepSize);
cannam@6:     m_begin = true;
cannam@6:     m_locked = false;
cannam@0: }
cannam@0: 
cannam@0: MatchVampPlugin::OutputList
cannam@0: MatchVampPlugin::getOutputDescriptors() const
cannam@0: {
cannam@0:     OutputList list;
cannam@0: 
cannam@0:     float outRate = 1.0 / 0.020; //!!! this is the default value of hopTime in Matcher
cannam@0: 
cannam@0:     OutputDescriptor desc;
cannam@0:     desc.identifier = "path";
cannam@0:     desc.name = "Path";
cannam@0:     desc.description = "Alignment path";
cannam@0:     desc.unit = "";
cannam@0:     desc.hasFixedBinCount = true;
cannam@0:     desc.binCount = 1;
cannam@0:     desc.hasKnownExtents = false;
cannam@0:     desc.isQuantized = true;
cannam@0:     desc.quantizeStep = 1;
cannam@0:     desc.sampleType = OutputDescriptor::VariableSampleRate;
cannam@0:     desc.sampleRate = outRate;
cannam@0:     list.push_back(desc);
cannam@0: 
cannam@0:     desc.identifier = "a_b";
cannam@0:     desc.name = "A-B Timeline";
cannam@0:     desc.description = "Timing in performance B corresponding to moments in performance A";
cannam@0:     desc.unit = "sec";
cannam@0:     desc.hasFixedBinCount = true;
cannam@0:     desc.binCount = 1;
cannam@0:     desc.hasKnownExtents = false;
cannam@0:     desc.isQuantized = false;
cannam@0:     desc.sampleType = OutputDescriptor::VariableSampleRate;
cannam@0:     desc.sampleRate = outRate;
cannam@0:     list.push_back(desc);
cannam@0: 
cannam@0:     desc.identifier = "b_a";
cannam@0:     desc.name = "B-A Timeline";
cannam@0:     desc.description = "Timing in performance A corresponding to moments in performance B";
cannam@0:     desc.unit = "sec";
cannam@0:     desc.hasFixedBinCount = true;
cannam@0:     desc.binCount = 1;
cannam@0:     desc.hasKnownExtents = false;
cannam@0:     desc.isQuantized = false;
cannam@0:     desc.sampleType = OutputDescriptor::VariableSampleRate;
cannam@0:     desc.sampleRate = outRate;
cannam@0:     list.push_back(desc);
cannam@0: 
cannam@0:     desc.identifier = "a_b_divergence";
cannam@0:     desc.name = "A-B Divergence";
cannam@0:     desc.description = "Difference between timings in performances A and B";
cannam@0:     desc.unit = "sec";
cannam@0:     desc.hasFixedBinCount = true;
cannam@0:     desc.binCount = 1;
cannam@0:     desc.hasKnownExtents = false;
cannam@0:     desc.isQuantized = false;
cannam@0:     desc.sampleType = OutputDescriptor::VariableSampleRate;
cannam@0:     desc.sampleRate = outRate;
cannam@0:     list.push_back(desc);
cannam@0: 
cannam@0:     desc.identifier = "a_b_temporatio";
cannam@0:     desc.name = "A-B Tempo Ratio";
cannam@0:     desc.description = "Ratio of tempi between performances A and B";
cannam@0:     desc.unit = "";
cannam@0:     desc.hasFixedBinCount = true;
cannam@0:     desc.binCount = 1;
cannam@0:     desc.hasKnownExtents = false;
cannam@0:     desc.isQuantized = false;
cannam@0:     desc.sampleType = OutputDescriptor::VariableSampleRate;
cannam@0:     desc.sampleRate = outRate;
cannam@0:     list.push_back(desc);
cannam@0: 
cannam@0:     return list;
cannam@0: }
cannam@0: 
cannam@0: MatchVampPlugin::FeatureSet
cannam@0: MatchVampPlugin::process(const float *const *inputBuffers,
cannam@0:                          Vamp::RealTime timestamp)
cannam@0: {
cannam@0:     if (m_begin) {
cannam@0:         if (!m_locked && m_serialise) {
cannam@0:             m_locked = true;
cannam@0: #ifdef _WIN32
cannam@0:             WaitForSingleObject(m_serialisingMutex, INFINITE);
cannam@0: #else
cannam@0:             pthread_mutex_lock(&m_serialisingMutex);
cannam@0: #endif
cannam@0:         }
cannam@0:         m_begin = false;
cannam@0:     }
cannam@0:     
cannam@0: //    std::cerr << timestamp.toString();
cannam@0: 
cannam@0:     feeder->feed(inputBuffers);
cannam@0: 
cannam@0: //    std::cerr << ".";
cannam@0: //    std::cerr << std::endl;
cannam@0: 
cannam@0:     return FeatureSet();
cannam@0: }
cannam@0: 
cannam@0: MatchVampPlugin::FeatureSet
cannam@0: MatchVampPlugin::getRemainingFeatures()
cannam@0: {
cannam@0:     int x = pm2->getFrameCount() - 1;
cannam@0:     int y = pm1->getFrameCount() - 1;
cannam@0: 
cannam@0:     Finder *finder = feeder->getFinder();
cannam@0: 
cannam@0:     std::vector<int> pathx;
cannam@0:     std::vector<int> pathy;
cannam@0: 
cannam@0: //    std::cerr << "initial x,y = " << x << std::endl;
cannam@0: 
cannam@0:     while (finder->find(y, x) && ((x > 0) || (y > 0))) {
cannam@0: 
cannam@0:         pathx.push_back(x);
cannam@0:         pathy.push_back(y);
cannam@0: 
cannam@0: //        std::cerr << pathx.size() << ": (" << x << "," << y << ")" << std::endl;
cannam@0: 
cannam@0:         switch (finder->getDistance() & ADVANCE_BOTH){
cannam@0:         case ADVANCE_THIS:  y--; break;
cannam@0:         case ADVANCE_OTHER: x--; break;
cannam@0:         case ADVANCE_BOTH:  x--; y--; break;
cannam@0:         default: // this would indicate a bug, but we wouldn't want to hang
cannam@0:             std::cerr << "WARNING: MatchVampPlugin::getRemainingFeatures: Neither matcher advanced in path backtrack at (" << x << "," << y << ")" << std::endl;
cannam@0:             if (x > y) x--; else y--; break;
cannam@0:         }
cannam@0:     }
cannam@0: 
cannam@0:     std::reverse(pathx.begin(), pathx.end());
cannam@0:     std::reverse(pathy.begin(), pathy.end());
cannam@0: 
cannam@0:     int smoothedLen = Path().smooth(pathx, pathy, pathx.size());
cannam@0: 
cannam@0:     FeatureSet returnFeatures;
cannam@0: 
cannam@0:     int prevx = 0;
cannam@0:     int prevy = 0;
cannam@0: 
cannam@0:     for (int i = 0; i < smoothedLen; ++i) {
cannam@0: 
cannam@0:         int x = pathx[i];
cannam@0:         int y = pathy[i];
cannam@0: 
cannam@0:         Vamp::RealTime xt = Vamp::RealTime::frame2RealTime
cannam@0:             (x * pm1->getHopSize(), lrintf(m_inputSampleRate));
cannam@0:         Vamp::RealTime yt = Vamp::RealTime::frame2RealTime
cannam@0:             (y * pm2->getHopSize(), lrintf(m_inputSampleRate));
cannam@0: 
cannam@0:         Feature feature;
cannam@0:         feature.hasTimestamp = true;
cannam@0:         feature.timestamp = xt;
cannam@0:         feature.values.clear();
cannam@0:         feature.values.push_back(yt.sec + double(yt.nsec)/1.0e9);
cannam@0:         returnFeatures[0].push_back(feature);
cannam@0:         
cannam@0:         if (x != prevx) {
cannam@0: 
cannam@0:             feature.hasTimestamp = true;
cannam@0:             feature.timestamp = xt;
cannam@0:             feature.values.clear();
cannam@0:             feature.values.push_back(yt.sec + yt.msec()/1000.0);
cannam@0:             returnFeatures[1].push_back(feature);
cannam@0: 
cannam@0:             Vamp::RealTime diff = yt - xt;
cannam@0:             feature.values.clear();
cannam@0:             feature.values.push_back(diff.sec + diff.msec()/1000.0);
cannam@0:             returnFeatures[3].push_back(feature);
cannam@0: 
cannam@0:             if (i > 0) {
cannam@0:                 int lookback = 100; //!!! arbitrary
cannam@0:                 if (lookback > i) lookback = i;
cannam@0:                 int xdiff = x - pathx[i-lookback];
cannam@0:                 int ydiff = y - pathy[i-lookback];
cannam@0:                 if (xdiff != 0 && ydiff != 0) {
cannam@0:                     float ratio = float(ydiff)/float(xdiff);
cannam@0:                     if (ratio < 8 && ratio > (1.0/8)) { //!!! just for now, since we aren't dealing properly with silence yet
cannam@0:                         feature.values.clear();
cannam@0:                         feature.values.push_back(ratio);
cannam@0:                         returnFeatures[4].push_back(feature);
cannam@0:                     }
cannam@0:                 }
cannam@0:             }
cannam@0:         }
cannam@0: 
cannam@0:         if (y != prevy) {
cannam@0:             feature.hasTimestamp = true;
cannam@0:             feature.timestamp = yt;
cannam@0:             feature.values.clear();
cannam@0:             feature.values.push_back(xt.sec + xt.msec()/1000.0);
cannam@0:             returnFeatures[2].push_back(feature);
cannam@0:         }
cannam@0: 
cannam@0:         prevx = x;
cannam@0:         prevy = y;
cannam@0:     }
cannam@0: 
cannam@0:     delete feeder;
cannam@0:     delete pm1;
cannam@0:     delete pm2;
cannam@0:     feeder = 0;
cannam@0:     pm1 = 0;
cannam@0:     pm2 = 0;
cannam@0: 
cannam@0:     if (m_locked) {
cannam@0: #ifdef _WIN32
cannam@0:         ReleaseMutex(m_serialisingMutex);
cannam@0: #else
cannam@0:         pthread_mutex_unlock(&m_serialisingMutex);
cannam@0: #endif
cannam@0:         m_locked = false;
cannam@0:     }
cannam@0: 
cannam@0:     return returnFeatures;
cannam@0:     
cannam@0: 
cannam@0: /*
cannam@0:     for (int i = 0; i < smoothedLen; ++i) {
cannam@0:         std::cerr << i << ": [" << pathx[i] << "," << pathy[i] << "]" << std::endl;
cannam@0:     }
cannam@0: 
cannam@0:     std::cerr << std::endl;
cannam@0:     std::cerr << "File: A" << std::endl;
cannam@0:     std::cerr << "Marks: -1" << std::endl;
cannam@0:     std::cerr << "FixedPoints: true 0" << std::endl;
cannam@0:     std::cerr << "0" << std::endl;
cannam@0:     std::cerr << "0" << std::endl;
cannam@0:     std::cerr << "0" << std::endl;
cannam@0:     std::cerr << "0" << std::endl;
cannam@0:     std::cerr << "File: B" << std::endl;
cannam@0:     std::cerr << "Marks: 0" << std::endl;
cannam@0:     std::cerr << "FixedPoints: true 0" << std::endl;
cannam@0:     std::cerr << "0.02" << std::endl;
cannam@0:     std::cerr << "0.02" << std::endl;
cannam@0: 
cannam@0:     std::cerr << smoothedLen << std::endl;
cannam@0:     for (int i = 0; i < smoothedLen; ++i) {
cannam@0:         std::cerr << pathx[i] << std::endl;
cannam@0:     }
cannam@0: 
cannam@0:     std::cerr << smoothedLen << std::endl;
cannam@0:     for (int i = 0; i < smoothedLen; ++i) {
cannam@0:         std::cerr << pathy[i] << std::endl;
cannam@0:     }
cannam@0: */
cannam@0: }
cannam@0: 
cannam@0: static Vamp::PluginAdapter<MatchVampPlugin> mvpAdapter;
cannam@0: 
cannam@0: const VampPluginDescriptor *vampGetPluginDescriptor(unsigned int version,
cannam@0:                                                     unsigned int index)
cannam@0: {
cannam@0:     if (version < 1) return 0;
cannam@0: 
cannam@0:     switch (index) {
cannam@0:     case  0: return mvpAdapter.getDescriptor();
cannam@0:     default: return 0;
cannam@0:     }
cannam@0: }