changeset 121:14a1664d1cfe adaptive_diagonals

Move adaptive smoothing over to Finder::smooth

--- a/src/Finder.cpp	Fri Dec 05 15:19:48 2014 +0000
+++ b/src/Finder.cpp	Fri Dec 05 17:28:53 2014 +0000
@@ -17,6 +17,7 @@
 #include "Finder.h"
 
 #include "Path.h"
+#include "MedianFilter.h"
 
 #include <algorithm>
 #include <iomanip>
@@ -439,16 +440,104 @@
 }
 
 void
-Finder::smoothWithPinPoints(const map<int, int> &pinpoints)
+Finder::smooth(const vector<double> &mag1, const vector<double> &mag2)
 {
+    if (m_m->getDiagonalWeight() <= 1.0) {
+        cerr << "Finder::smooth: Diagonal weight is already "
+             << m_m->getDiagonalWeight() << ", adaptive smoothing will have "
+             << "no effect, skipping it" << endl;
+        return;
+    }
+
+    vector<double> confidence;
+
+    vector<int> pathx, pathy;
+    vector<float> distances;
+    retrievePath(pathx, pathy, distances);
+
+    int len = pathx.size();
+    
+    for (int i = 0; i < len; ++i) {
+
+        int x = pathx[i];
+        int y = pathy[i];
+
+        double magSum = mag1[x] + mag2[y];
+        double distance = distances[i];
+        double c = magSum - distance * magSum;
+        confidence.push_back(c);
+
+        /*
+        if (x != prevx) {
+            Feature f;
+            f.values.push_back(magSum);
+            returnFeatures[m_magOutNo].push_back(f);
+            
+            f.values.clear();
+            f.values.push_back(distance);
+            returnFeatures[m_distOutNo].push_back(f);
+        }
+        
+        prevx = x;
+        prevy = y;
+        */
+    }
+
+    confidence = MedianFilter<double>::filter(3, confidence);
+    vector<double> filtered = MedianFilter<double>::filter(50, confidence);
+    for (int i = 0; i < len; ++i) {
+        confidence[i] -= filtered[i];
+        if (confidence[i] < 0.f) {
+            confidence[i] = 0.f;
+        }
+    }
+    vector<double> deriv;
+    deriv.resize(len, 0.f);
+    for (int i = 1; i < len; ++i) {
+        deriv[i] = confidence[i] - confidence[i-1];
+    }
+    vector<int> inflections;
+    for (int i = 1; i < len; ++i) {
+        if (deriv[i-1] > 0 && deriv[i] < 0) {
+            inflections.push_back(i);
+        }
+    }
+        
+    /*    
+    for (int i = 0; i < len; ++i) {
+        
+        int x = pathx[i];
+        int y = pathy[i];
+
+        if (x != prevx) {
+            Feature f;
+            double c = confidence[i];
+            f.values.push_back(c);
+            returnFeatures[m_confidenceOutNo].push_back(f);
+        }
+        
+        prevx = x;
+        prevy = y;
+    }
+    */
+    
+    map<int, int> pinpoints;
+            
+    for (int ii = 0; ii < int(inflections.size()); ++ii) {
+
+        int i = inflections[ii];
+            
+        int x = pathx[i];
+        int y = pathy[i];
+
+        pinpoints[x] = y;
+    }
+
     cerr << "Pin points are:" << endl;
-
-    typedef map<int, int> PPMap;
-
-    for (PPMap::const_iterator i = pinpoints.begin(); i != pinpoints.end(); ++i) {
+    for (map<int, int>::const_iterator i = pinpoints.begin();
+         i != pinpoints.end(); ++i) {
         cerr << "[" << i->first << "," << i->second << "] ";
     }
-    
     cerr << endl;
 
     if (pinpoints.size() < 2) return;
@@ -458,7 +547,9 @@
     
     pair<int, int> prev = *pinpoints.begin();
 
-    for (PPMap::const_iterator i = pinpoints.begin(); i != pinpoints.end(); ++i) {
+    for (map<int, int>::const_iterator i = pinpoints.begin();
+         i != pinpoints.end(); ++i) {
+
         if (i == pinpoints.begin()) continue;
 
         pair<int, int> curr = *i;

--- a/src/Finder.h	Fri Dec 05 15:19:48 2014 +0000
+++ b/src/Finder.h	Fri Dec 05 17:28:53 2014 +0000
@@ -55,6 +55,14 @@
     void recalculatePathCostMatrix(int r1, int c1, int r2, int c2);
 
     /**
+     * Recalculate a rectangle of the path cost matrix using the given
+     * diagonal weight instead of the weight obtained from the Matcher
+     * parameters.
+     */
+    void recalculatePathCostMatrix(int r1, int c1, int r2, int c2,
+                                   float diagonalWeight);
+
+    /**
      * Track back after all of the matchers have been fed in order to
      * obtain the lowest cost path available. Path x and y coordinate
      * pairs are returned in corresponding elements of pathx and
@@ -66,7 +74,21 @@
                       std::vector<int> &pathy,
                       std::vector<float> &distances);
 
-    void smoothWithPinPoints(const std::map<int, int> &);
+    /**
+     * Using the provided magnitude arrays for the first and second
+     * inputs as an indication of salience, smooth the cost matrix by
+     * recalculating with locally adaptive diagonal weights in order
+     * to obtain a smoother path that retains the original locations
+     * for the most salient points. Subsequent calls to retrievePath
+     * will retrieve the smoothed version. The original can be
+     * restored with a call to recalculatePathCostMatrix().
+     *
+     * Note that this is quite separate from Path::smooth() which
+     * smooths a path without trying to maintain locations for salient
+     * points. That function will achieve a smoother path, but may
+     * smooth out significant locations such as onsets.
+     */
+    void smooth(const vector<double> &mag1, const vector<double> &mag2);
     
 protected:
 #ifdef PERFORM_ERROR_CHECKS
@@ -87,9 +109,6 @@
     void checkAndReport();
 #endif
 
-    void recalculatePathCostMatrix(int r1, int c1, int r2, int c2,
-                                   float diagonalWeight);
-
     void getEndPoint(int &x, int &y);
     
     Matcher *m_m;

--- a/src/MatchPipeline.cpp	Fri Dec 05 15:19:48 2014 +0000
+++ b/src/MatchPipeline.cpp	Fri Dec 05 17:28:53 2014 +0000
@@ -72,6 +72,23 @@
     f2 = m_f2;
 }
 
+static double
+magOf(const vector<double> &f)
+{
+    double mag = 0.0;
+    for (int j = 0; j < (int)f.size(); ++j) {
+        mag += f[j] * f[j];
+    }
+    return sqrt(mag);
+}
+
+void
+MatchPipeline::extractFeatureMagnitudes(double &mag1, double &mag2)
+{
+    mag1 = magOf(m_f1);
+    mag2 = magOf(m_f2);
+}
+
 void
 MatchPipeline::extractConditionedFeatures(vector<double> &c1, vector<double> &c2)
 {

--- a/src/MatchPipeline.h	Fri Dec 05 15:19:48 2014 +0000
+++ b/src/MatchPipeline.h	Fri Dec 05 17:28:53 2014 +0000
@@ -74,11 +74,21 @@
      * If a frame was just fed in at the first or second pipeline
      * stage, it can be retrieved from the second stage here. That is,
      * if you provided frequency-domain audio, extractFeatures will
-     * give you back the FeatureExtractor's features.
+     * give you back the FeatureExtractor's features; if you provided
+     * features, extractFeatures will simply give you those back. If
+     * you provided conditioned features, the return from this
+     * function is undefined.
      */
     void extractFeatures(vector<double> &f1, vector<double> &f2);
 
     /**
+     * If a frame was just fed in at the first or second pipeline
+     * stage, you can obtain the magnitudes of its features for both
+     * inputs here.
+     */
+    void extractFeatureMagnitudes(double &mag1, double &mag2);
+    
+    /**
      * Retrieve the conditioned features from the third pipeline stage.
      */
     void extractConditionedFeatures(vector<double> &f1, vector<double> &f2);

--- a/src/MatchVampPlugin.cpp	Fri Dec 05 15:19:48 2014 +0000
+++ b/src/MatchVampPlugin.cpp	Fri Dec 05 17:28:53 2014 +0000
@@ -19,8 +19,6 @@
 #include "Matcher.h"
 #include "MatchFeatureFeeder.h"
 #include "FeatureExtractor.h"
-#include "Path.h"
-#include "MedianFilter.h"
 
 #include <vamp/vamp.h>
 #include <vamp-sdk/PluginAdapter.h>
@@ -518,7 +516,7 @@
     desc.sampleRate = outRate;
     m_magOutNo = list.size();
     list.push_back(desc);
-
+/*
     desc.identifier = "confidence";
     desc.name = "Confidence";
     desc.description = "Confidence metric for the quality of match at each point-in-A along the chosen alignment path";
@@ -544,21 +542,10 @@
     desc.sampleRate = outRate;
     m_confPeakOutNo = list.size();
     list.push_back(desc);
-
+*/
     return list;
 }
 
-static float
-magOf(const vector<double> &f)
-{
-    double mag = 0.0;
-    for (int j = 0; j < (int)f.size(); ++j) {
-        mag += f[j] * f[j];
-    }
-    mag = sqrt(mag);
-    return float(mag);
-}
-
 MatchVampPlugin::FeatureSet
 MatchVampPlugin::process(const float *const *inputBuffers,
                          Vamp::RealTime timestamp)
@@ -585,10 +572,10 @@
     m_pipeline->extractFeatures(f1, f2);
     m_pipeline->extractConditionedFeatures(c1, c2);    
 
-    m_mag1.push_back(magOf(f1));
-    m_mag2.push_back(magOf(f2));
-    m_cmag1.push_back(magOf(c1));
-    m_cmag2.push_back(magOf(c2));
+    double m1, m2;
+    m_pipeline->extractFeatureMagnitudes(m1, m2);
+    m_mag1.push_back(m1);
+    m_mag2.push_back(m2);
 
     FeatureSet returnFeatures;
 
@@ -635,6 +622,11 @@
     FeatureSet returnFeatures;
     
     Finder *finder = m_pipeline->getFinder();
+
+    if (m_smooth) {
+        finder->smooth(m_mag1, m_mag2);
+    }
+
     vector<int> pathx;
     vector<int> pathy;
     vector<float> distances;
@@ -644,100 +636,6 @@
     int prevy = 0;
     int len = pathx.size();
 
-//!!!
-//  m_smooth = true;
-    
-    if (m_smooth) {
-    
-        vector<float> confidence;
-
-        for (int i = 0; i < len; ++i) {
-            int x = pathx[i];
-            int y = pathy[i];
-
-            float magSum = m_mag1[x] + m_mag2[y];
-            float distance = distances[i];
-            float c = magSum - distance * magSum;
-            confidence.push_back(c);
-
-            if (x != prevx) {
-                Feature f;
-                f.values.push_back(magSum);
-                returnFeatures[m_magOutNo].push_back(f);
-
-                f.values.clear();
-                f.values.push_back(distance);
-                returnFeatures[m_distOutNo].push_back(f);
-            }
-
-            prevx = x;
-            prevy = y;
-        }
-
-        confidence = MedianFilter<float>::filter(3, confidence);
-        vector<float> filtered = MedianFilter<float>::filter(50, confidence);
-        for (int i = 0; i < len; ++i) {
-            confidence[i] -= filtered[i];
-            if (confidence[i] < 0.f) {
-                confidence[i] = 0.f;
-            }
-        }
-        vector<float> deriv;
-        deriv.resize(len, 0.f);
-        for (int i = 1; i < len; ++i) {
-            deriv[i] = confidence[i] - confidence[i-1];
-        }
-        vector<int> inflections;
-        for (int i = 1; i < len; ++i) {
-            if (deriv[i-1] > 0 && deriv[i] < 0) {
-                inflections.push_back(i);
-            }
-        }
-        
-        for (int i = 0; i < len; ++i) {
-
-            int x = pathx[i];
-            int y = pathy[i];
-
-            if (x != prevx) {
-                Feature f;
-                float c = confidence[i];
-                f.values.push_back(c);
-                returnFeatures[m_confidenceOutNo].push_back(f);
-            }
-
-            prevx = x;
-            prevy = y;
-        }
-
-        map<int, int> pinpoints;
-            
-        for (int ii = 0; ii < int(inflections.size()); ++ii) {
-
-            int i = inflections[ii];
-            
-            int x = pathx[i];
-            int y = pathy[i];
-
-            pinpoints[x] = y;
-                
-            Vamp::RealTime xt = Vamp::RealTime::frame2RealTime
-                (x * m_stepSize, lrintf(m_inputSampleRate));
-            Feature feature;
-            feature.hasTimestamp = true;
-            feature.timestamp = m_startTime + xt;
-            returnFeatures[m_confPeakOutNo].push_back(feature);
-        }
-
-        finder->smoothWithPinPoints(pinpoints);
-
-        pathx.clear();
-        pathy.clear();
-        distances.clear();
-        finder->retrievePath(pathx, pathy, distances);
-        len = pathx.size();
-    }    
-
     for (int i = 0; i < len; ++i) {
 
         int x = pathx[i];

--- a/src/MatchVampPlugin.h	Fri Dec 05 15:19:48 2014 +0000
+++ b/src/MatchVampPlugin.h	Fri Dec 05 17:28:53 2014 +0000
@@ -90,10 +90,8 @@
     FeatureConditioner::Parameters m_fcParams;
     FeatureConditioner::Parameters m_defaultFcParams;
 
-    std::vector<float> m_mag1;
-    std::vector<float> m_mag2;
-    std::vector<float> m_cmag1;
-    std::vector<float> m_cmag2;
+    std::vector<double> m_mag1;
+    std::vector<double> m_mag2;
     
     mutable int m_pathOutNo;
     mutable int m_abOutNo;
@@ -106,9 +104,11 @@
     mutable int m_bFeaturesOutNo;
     mutable int m_magOutNo;
     mutable int m_distOutNo;
+/*
     mutable int m_confidenceOutNo;
     mutable int m_confPeakOutNo;
-
+*/
+    
 #ifdef _WIN32
     static HANDLE m_serialisingMutex;
 #else