Mercurial > hg > silvet

changeset 314:f98ba4f47e49 livemode

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge from default branch
author Chris Cannam
date Tue, 28 Apr 2015 11:24:23 +0100
parents 26be240475b5 (current diff) fa2ffbb786df (diff)
children dec47312ed40
files .hgsubstate Makefile.inc Makefile.linux src/EM.cpp src/Silvet.cpp src/Silvet.h
diffstat 6 files changed, 172 insertions(+), 73 deletions(-) [+]
line wrap: on
line diff
--- a/.hgsubstate	Mon Jan 19 09:33:35 2015 +0000
+++ b/.hgsubstate	Tue Apr 28 11:24:23 2015 +0100
@@ -1,3 +1,3 @@
-352579430f81c79a550741a97bb1f96fea05da75 bqvec
-4aa3d343f5fcc21c821c0f2f633980da2d8fc8bd constant-q-cpp
+7a48704e9a0fac1486240f9f7b7e31436a588064 bqvec
+4d109d855c671c514dbc4947b0dae7cada112d8c constant-q-cpp
 d25a2e91e9d84aaff25e5d746398232d182d127d flattendynamics
--- a/Makefile.inc	Mon Jan 19 09:33:35 2015 +0000
+++ b/Makefile.inc	Tue Apr 28 11:24:23 2015 +0100
@@ -13,7 +13,7 @@
 CC	?= gcc
 
 CFLAGS := $(CFLAGS)
-CXXFLAGS := $(CFLAGS) -I. -I$(VAMPSDK_DIR) -I$(CQ_DIR) -I$(BQVEC_DIR) -I$(FD_DIR) $(CXXFLAGS)
+CXXFLAGS := $(CFLAGS) -I. -I$(VAMPSDK_DIR) -I$(CQ_DIR) -I$(BQVEC_DIR) -I$(BQVEC_DIR) -I$(FD_DIR) $(CXXFLAGS)
 
 LDFLAGS := $(LDFLAGS) 
 PLUGIN_LDFLAGS := $(LDFLAGS) $(PLUGIN_LDFLAGS)
@@ -23,7 +23,7 @@
 PLUGIN_HEADERS := $(SRC_DIR)/Silvet.h $(SRC_DIR)/EM.h $(SRC_DIR)/Instruments.h $(SRC_DIR)/LiveInstruments.h
 PLUGIN_SOURCES := $(SRC_DIR)/Silvet.cpp $(SRC_DIR)/EM.cpp $(SRC_DIR)/Instruments.cpp $(SRC_DIR)/LiveInstruments.cpp $(SRC_DIR)/libmain.cpp
 
-BQVEC_HEADERS	:= $(BQVEC_DIR)/bqvec/Allocators.h $(BQVEC_DIR)/bqvec/Restrict.h $(BQVEC_DIR)/bqvec/VectorOps.h
+BQVEC_HEADERS	:= $(BQVEC_DIR)/Allocators.h $(BQVEC_DIR)/Restrict.h $(BQVEC_DIR)/VectorOps.h
 BQVEC_SOURCES	:= $(BQVEC_DIR)/src/Allocators.cpp
 
 FD_HEADERS	:= $(FD_DIR)/flattendynamics-ladspa.h
@@ -81,5 +81,3 @@
 flattendynamics/flattendynamics-ladspa.o: flattendynamics/flattendynamics-ladspa.h
 src/Silvet.o: src/MedianFilter.h src/Instruments.h
 src/LiveInstruments.o: src/Instruments.h
-bqvec/bqvec/Allocators.o: bqvec/bqvec/VectorOps.h bqvec/bqvec/Restrict.h
-bqvec/bqvec/VectorOps.o: bqvec/bqvec/Restrict.h
--- a/Makefile.linux	Mon Jan 19 09:33:35 2015 +0000
+++ b/Makefile.linux	Tue Apr 28 11:24:23 2015 +0100
@@ -1,12 +1,12 @@
 
-CFLAGS := -Wall -O3 -ffast-math -msse -msse2 -mfpmath=sse -ftree-vectorize -fPIC -I../vamp-plugin-sdk/ -DUSE_PTHREADS
+CFLAGS := -Wall -O3 -ffast-math -msse -msse2 -mfpmath=sse -ftree-vectorize -fPIC -I../vamp-plugin-sdk/ 
 
 #CFLAGS := -g -fPIC -I../vamp-plugin-sdk
 
-CXXFLAGS := $(CFLAGS)
+CXXFLAGS := $(CFLAGS) -std=c++11
 
 VAMPSDK_DIR := ../vamp-plugin-sdk
-PLUGIN_LDFLAGS := -shared -Wl,-Bsymbolic -Wl,-z,defs -Wl,--version-script=vamp-plugin.map -lpthread
+PLUGIN_LDFLAGS := -shared -Wl,-Bsymbolic -Wl,-z,defs -Wl,--version-script=vamp-plugin.map 
 
 PLUGIN_EXT := .so
 
--- a/src/EM.cpp	Mon Jan 19 09:33:35 2015 +0000
+++ b/src/EM.cpp	Tue Apr 28 11:24:23 2015 +0100
@@ -22,7 +22,6 @@
 
 #include "bqvec/VectorOps.h"
 #include "bqvec/Allocators.h"
-
 #include "Instruments.h"
 
 using std::vector;
--- a/src/Silvet.cpp	Mon Jan 19 09:33:35 2015 +0000
+++ b/src/Silvet.cpp	Tue Apr 28 11:24:23 2015 +0100
@@ -24,6 +24,7 @@
 #include "LiveInstruments.h"
 
 #include <vector>
+#include <future>
 
 #include <cstdio>
 
@@ -31,6 +32,9 @@
 using std::cout;
 using std::cerr;
 using std::endl;
+using std::pair;
+using std::future;
+using std::async;
 using Vamp::RealTime;
 
 static int processingSampleRate = 44100;
@@ -51,7 +55,8 @@
     m_mode(HighQualityMode),
     m_fineTuning(false),
     m_instrument(0),
-    m_colsPerSec(50)
+    m_colsPerSec(50),
+    m_haveStartTime(false)
 {
 }
 
@@ -295,6 +300,26 @@
     m_pitchOutputNo = list.size();
     list.push_back(d);
 
+    d.identifier = "chroma";
+    d.name = "Pitch chroma distribution";
+    d.description = "Pitch chroma distribution formed by wrapping the un-thresholded pitch activation distribution into a single octave of semitone bins.";
+    d.unit = "";
+    d.hasFixedBinCount = true;
+    d.binCount = 12;
+    d.binNames.clear();
+    if (m_cq) {
+        for (int i = 0; i < 12; ++i) {
+            d.binNames.push_back(chromaName(i));
+        }
+    }
+    d.hasKnownExtents = false;
+    d.isQuantized = false;
+    d.sampleType = OutputDescriptor::FixedSampleRate;
+    d.sampleRate = m_colsPerSec;
+    d.hasDuration = false;
+    m_chromaOutputNo = list.size();
+    list.push_back(d);
+
     d.identifier = "templates";
     d.name = "Templates";
     d.description = "Constant-Q spectral templates for the selected instrument pack.";
@@ -328,13 +353,19 @@
 }
 
 std::string
-Silvet::noteName(int note, int shift, int shiftCount) const
+Silvet::chromaName(int pitch) const
 {
     static const char *names[] = {
         "A", "A#", "B", "C", "C#", "D", "D#", "E", "F", "F#", "G", "G#"
     };
 
-    const char *n = names[note % 12];
+    return names[pitch];
+}
+    
+std::string
+Silvet::noteName(int note, int shift, int shiftCount) const
+{
+    string n = chromaName(note % 12);
 
     int oct = (note + 9) / 12; 
     
@@ -348,11 +379,11 @@
     }
 
     if (pshift > 0.f) {
-        sprintf(buf, "%s%d+%dc", n, oct, int(round(pshift * 100)));
+        sprintf(buf, "%s%d+%dc", n.c_str(), oct, int(round(pshift * 100)));
     } else if (pshift < 0.f) {
-        sprintf(buf, "%s%d-%dc", n, oct, int(round((-pshift) * 100)));
+        sprintf(buf, "%s%d-%dc", n.c_str(), oct, int(round((-pshift) * 100)));
     } else {
-        sprintf(buf, "%s%d", n, oct);
+        sprintf(buf, "%s%d", n.c_str(), oct);
     }
 
     return buf;
@@ -492,6 +523,7 @@
     m_columnCount = 0;
     m_resampledCount = 0;
     m_startTime = RealTime::zeroTime;
+    m_haveStartTime = false;
 }
 
 Silvet::FeatureSet
@@ -499,8 +531,11 @@
 {
     FeatureSet fs;
     
-    if (m_columnCount == 0) {
+    if (!m_haveStartTime) {
+
         m_startTime = timestamp;
+        m_haveStartTime = true;
+
         insertTemplateFeatures(fs);
     }
 
@@ -597,9 +632,7 @@
 
     int width = filtered.size();
 
-    int iterations = (m_mode == HighQualityMode ? 20 : 10);
-
-    Grid localPitches(width, vector<double>(pack.templateNoteCount, 0.0));
+    Grid localPitches(width);
 
     bool wantShifts = (m_mode == HighQualityMode) && m_fineTuning;
     int shiftCount = 1;
@@ -609,68 +642,63 @@
 
     vector<vector<int> > localBestShifts;
     if (wantShifts) {
-        localBestShifts = 
-            vector<vector<int> >(width, vector<int>(pack.templateNoteCount, 0));
+        localBestShifts = vector<vector<int> >(width);
     }
 
-    double columnThreshold = 1e-5;
+#ifndef MAX_EM_THREADS
+#define MAX_EM_THREADS 8
+#endif
 
-    if (m_mode == LiveMode) {
-        columnThreshold /= 20;
+#if (defined(MAX_EM_THREADS) && (MAX_EM_THREADS > 1))
+    for (int i = 0; i < width; ) {
+        typedef future<pair<vector<double>, vector<int>>> EMFuture;
+        vector<EMFuture> results;
+        for (int j = 0; j < MAX_EM_THREADS && i + j < width; ++j) {
+            results.push_back
+                (async(std::launch::async,
+                       [&](int index) {
+                           return applyEM(pack, filtered.at(index), wantShifts);
+                       }, i + j));
+        }
+        for (int j = 0; j < MAX_EM_THREADS && i + j < width; ++j) {
+            auto out = results[j].get();
+            localPitches[i+j] = out.first;
+            if (wantShifts) localBestShifts[i+j] = out.second;
+        }
+        i += MAX_EM_THREADS;
     }
-    
-#pragma omp parallel for
+#else
     for (int i = 0; i < width; ++i) {
-
-        double sum = 0.0;
-        for (int j = 0; j < pack.templateHeight; ++j) {
-            sum += filtered.at(i).at(j);
-        }
-        if (sum < columnThreshold) continue;
-
-        EM em(&pack, m_mode == HighQualityMode);
-
-        em.setPitchSparsity(pack.pitchSparsity);
-        em.setSourceSparsity(pack.sourceSparsity);
-
-        for (int j = 0; j < iterations; ++j) {
-            em.iterate(filtered.at(i).data());
-        }
-
-        const float *pitchDist = em.getPitchDistribution();
-        const float *const *shiftDist = em.getShifts();
-
-        for (int j = 0; j < pack.templateNoteCount; ++j) {
-
-            localPitches[i][j] = pitchDist[j] * sum;
-
-            int bestShift = 0;
-            float bestShiftValue = 0.0;
-            if (wantShifts) {
-                for (int k = 0; k < shiftCount; ++k) {
-                    float value = shiftDist[k][j];
-                    if (k == 0 || value > bestShiftValue) {
-                        bestShiftValue = value;
-                        bestShift = k;
-                    }
-                }
-                localBestShifts[i][j] = bestShift;
-            }                
-        }
+        auto out = applyEM(pack, filtered.at(i), wantShifts);
+        localPitches[i] = out.first;
+        if (wantShifts) localBestShifts[i] = out.second;
     }
+#endif
         
     for (int i = 0; i < width; ++i) {
 
+        // This returns a filtered column, and pushes the
+        // up-to-max-polyphony activation column to m_pianoRoll
         vector<double> filtered = postProcess
             (localPitches[i], localBestShifts[i], wantShifts);
 
+        RealTime timestamp = getColumnTimestamp(m_pianoRoll.size() - 1);
+        float inputGain = getInputGainAt(timestamp);
+
         Feature f;
         for (int j = 0; j < (int)filtered.size(); ++j) {
-            float v(filtered[j]);
+            float v = filtered[j];
             if (v < pack.levelThreshold) v = 0.f;
-            f.values.push_back(v);
+            f.values.push_back(v / inputGain);
         }
         fs[m_pitchOutputNo].push_back(f);
+
+        f.values.clear();
+        f.values.resize(12);
+        for (int j = 0; j < (int)filtered.size(); ++j) {
+            f.values[j % 12] += filtered[j] / inputGain;
+        }
+        fs[m_chromaOutputNo].push_back(f);
         
         FeatureList noteFeatures = noteTrack(shiftCount);
 
@@ -681,6 +709,66 @@
     }
 }
 
+pair<vector<double>, vector<int> >
+Silvet::applyEM(const InstrumentPack &pack,
+                const vector<double> &column,
+                bool wantShifts)
+{
+    double columnThreshold = 1e-5;
+    
+    if (m_mode == LiveMode) {
+        columnThreshold /= 20;
+    }
+    
+    vector<double> pitches(pack.templateNoteCount, 0.0);
+    vector<int> bestShifts;
+    
+    double sum = 0.0;
+    for (int j = 0; j < pack.templateHeight; ++j) {
+        sum += column.at(j);
+    }
+    if (sum < columnThreshold) return { pitches, bestShifts };
+
+    EM em(&pack, m_mode == HighQualityMode);
+
+    em.setPitchSparsity(pack.pitchSparsity);
+    em.setSourceSparsity(pack.sourceSparsity);
+
+    int iterations = (m_mode == HighQualityMode ? 20 : 10);
+
+    for (int j = 0; j < iterations; ++j) {
+        em.iterate(column.data());
+    }
+
+    const float *pitchDist = em.getPitchDistribution();
+    const float *const *shiftDist = em.getShifts();
+
+    int shiftCount = 1;
+    if (wantShifts) {
+        shiftCount = pack.templateMaxShift * 2 + 1;
+    }
+    
+    for (int j = 0; j < pack.templateNoteCount; ++j) {
+
+        pitches[j] = pitchDist[j] * sum;
+
+        int bestShift = 0;
+        float bestShiftValue = 0.0;
+        if (wantShifts) {
+            for (int k = 0; k < shiftCount; ++k) {
+                float value = shiftDist[k][j];
+                if (k == 0 || value > bestShiftValue) {
+                    bestShiftValue = value;
+                    bestShift = k;
+                }
+            }
+            bestShifts.push_back(bestShift);
+        }                
+    }
+
+    return { pitches, bestShifts };
+}
+
 Silvet::Grid
 Silvet::preProcess(const Grid &in)
 {
@@ -943,6 +1031,16 @@
     }
 }
 
+RealTime
+Silvet::getColumnTimestamp(int column)
+{
+    double columnDuration = 1.0 / m_colsPerSec;
+    int postFilterLatency = int(m_postFilter[0]->getSize() / 2);
+
+    return m_startTime + RealTime::fromSeconds
+        (columnDuration * (column - postFilterLatency) + 0.02);
+}
+
 Silvet::Feature
 Silvet::makeNoteFeature(int start,
                         int end,
@@ -951,18 +1049,13 @@
                         int shiftCount,
                         int velocity)
 {
-    double columnDuration = 1.0 / m_colsPerSec;
-    int postFilterLatency = int(m_postFilter[0]->getSize() / 2);
-
     Feature f;
 
     f.hasTimestamp = true;
-    f.timestamp = m_startTime + RealTime::fromSeconds
-        (columnDuration * (start - postFilterLatency) + 0.02);
+    f.timestamp = getColumnTimestamp(start);
 
     f.hasDuration = true;
-    f.duration = RealTime::fromSeconds
-        (columnDuration * (end - start));
+    f.duration = getColumnTimestamp(end) - f.timestamp;
 
     f.values.clear();
 
--- a/src/Silvet.h	Mon Jan 19 09:33:35 2015 +0000
+++ b/src/Silvet.h	Tue Apr 28 11:24:23 2015 +0100
@@ -107,6 +107,10 @@
 
     Grid preProcess(const Grid &);
 
+    std::pair<vector<double>, vector<int> > applyEM(const InstrumentPack &pack,
+                                                    const vector<double> &column,
+                                                    bool wantShifts);
+    
     vector<double> postProcess(const vector<double> &pitches,
                                const vector<int> &bestShifts,
                                bool wantShifts); // -> piano roll column
@@ -115,7 +119,9 @@
 
     void emitNote(int start, int end, int note, int shiftCount,
                   FeatureList &noteFeatures);
-
+    
+    Vamp::RealTime getColumnTimestamp(int column);
+    
     Feature makeNoteFeature(int start, int end, int note, int shift,
                             int shiftCount, int velocity);
 
@@ -125,6 +131,7 @@
     
     void transcribe(const Grid &, FeatureSet &);
 
+    string chromaName(int n) const;
     string noteName(int n, int shift, int shiftCount) const;
     float noteFrequency(int n, int shift, int shiftCount) const;
 
@@ -132,11 +139,13 @@
     int m_columnCount;
     int m_resampledCount;
     Vamp::RealTime m_startTime;
+    bool m_haveStartTime;
 
     mutable int m_notesOutputNo;
     mutable int m_fcqOutputNo;
     mutable int m_pitchOutputNo;
     mutable int m_templateOutputNo;
+    mutable int m_chromaOutputNo;
 };
 
 #endif