Chris@31: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@31: 
Chris@31: /*
Chris@31:   Silvet
Chris@31: 
Chris@31:   A Vamp plugin for note transcription.
Chris@31:   Centre for Digital Music, Queen Mary University of London.
Chris@31:     
Chris@31:   This program is free software; you can redistribute it and/or
Chris@31:   modify it under the terms of the GNU General Public License as
Chris@31:   published by the Free Software Foundation; either version 2 of the
Chris@31:   License, or (at your option) any later version.  See the file
Chris@31:   COPYING included with this distribution for more information.
Chris@31: */
Chris@31: 
Chris@31: #include "Silvet.h"
Chris@34: #include "EM.h"
Chris@31: 
Chris@152: #include <cq/CQSpectrogram.h>
Chris@31: 
Chris@152: #include "MedianFilter.h"
Chris@152: #include "constant-q-cpp/src/dsp/Resampler.h"
Chris@246: #include "flattendynamics-ladspa.h"
Chris@298: #include "LiveInstruments.h"
Chris@31: 
Chris@31: #include <vector>
Chris@31: 
Chris@32: #include <cstdio>
Chris@32: 
Chris@356: #if (defined(MAX_EM_THREADS) && (MAX_EM_THREADS > 1))
Chris@356: #include <future>
Chris@356: using std::future;
Chris@356: using std::async;
Chris@356: #endif
Chris@356: 
Chris@31: using std::vector;
Chris@48: using std::cout;
Chris@31: using std::cerr;
Chris@31: using std::endl;
Chris@311: using std::pair;
Chris@356: 
Chris@40: using Vamp::RealTime;
Chris@31: 
Chris@31: static int processingSampleRate = 44100;
Chris@298: 
Chris@298: static int binsPerSemitoneLive = 1;
Chris@298: static int binsPerSemitoneNormal = 5;
Chris@170: 
Chris@272: static int minInputSampleRate = 100;
Chris@272: static int maxInputSampleRate = 192000;
Chris@272: 
Chris@316: static const Silvet::ProcessingMode defaultMode = Silvet::HighQualityMode;
Chris@316: 
Chris@31: Silvet::Silvet(float inputSampleRate) :
Chris@31:     Plugin(inputSampleRate),
Chris@161:     m_instruments(InstrumentPack::listInstrumentPacks()),
Chris@298:     m_liveInstruments(LiveAdapter::adaptAll(m_instruments)),
Chris@31:     m_resampler(0),
Chris@246:     m_flattener(0),
Chris@110:     m_cq(0),
Chris@316:     m_mode(defaultMode),
Chris@166:     m_fineTuning(false),
Chris@178:     m_instrument(0),
Chris@313:     m_colsPerSec(50),
Chris@313:     m_haveStartTime(false)
Chris@31: {
Chris@31: }
Chris@31: 
Chris@31: Silvet::~Silvet()
Chris@31: {
Chris@31:     delete m_resampler;
Chris@246:     delete m_flattener;
Chris@31:     delete m_cq;
Chris@41:     for (int i = 0; i < (int)m_postFilter.size(); ++i) {
Chris@41:         delete m_postFilter[i];
Chris@41:     }
Chris@31: }
Chris@31: 
Chris@31: string
Chris@31: Silvet::getIdentifier() const
Chris@31: {
Chris@31:     return "silvet";
Chris@31: }
Chris@31: 
Chris@31: string
Chris@31: Silvet::getName() const
Chris@31: {
Chris@31:     return "Silvet Note Transcription";
Chris@31: }
Chris@31: 
Chris@31: string
Chris@31: Silvet::getDescription() const
Chris@31: {
Chris@191:     return "Estimate the note onsets, pitches, and durations that make up a music recording.";
Chris@31: }
Chris@31: 
Chris@31: string
Chris@31: Silvet::getMaker() const
Chris@31: {
Chris@191:     return "Queen Mary, University of London";
Chris@31: }
Chris@31: 
Chris@31: int
Chris@31: Silvet::getPluginVersion() const
Chris@31: {
Chris@309:     return 3;
Chris@31: }
Chris@31: 
Chris@31: string
Chris@31: Silvet::getCopyright() const
Chris@31: {
Chris@191:     return "Method by Emmanouil Benetos and Simon Dixon; plugin by Chris Cannam and Emmanouil Benetos. GPL licence.";
Chris@31: }
Chris@31: 
Chris@31: Silvet::InputDomain
Chris@31: Silvet::getInputDomain() const
Chris@31: {
Chris@31:     return TimeDomain;
Chris@31: }
Chris@31: 
Chris@31: size_t
Chris@31: Silvet::getPreferredBlockSize() const
Chris@31: {
Chris@31:     return 0;
Chris@31: }
Chris@31: 
Chris@31: size_t 
Chris@31: Silvet::getPreferredStepSize() const
Chris@31: {
Chris@31:     return 0;
Chris@31: }
Chris@31: 
Chris@31: size_t
Chris@31: Silvet::getMinChannelCount() const
Chris@31: {
Chris@31:     return 1;
Chris@31: }
Chris@31: 
Chris@31: size_t
Chris@31: Silvet::getMaxChannelCount() const
Chris@31: {
Chris@31:     return 1;
Chris@31: }
Chris@31: 
Chris@31: Silvet::ParameterList
Chris@31: Silvet::getParameterDescriptors() const
Chris@31: {
Chris@31:     ParameterList list;
Chris@110: 
Chris@110:     ParameterDescriptor desc;
Chris@110:     desc.identifier = "mode";
Chris@110:     desc.name = "Processing mode";
Chris@110:     desc.unit = "";
Chris@341:     desc.description = "Sets the tradeoff of processing speed against transcription quality. Live mode is much faster and detects notes with relatively low latency; Intensive mode (the default) is slower but will almost always produce better results.";
Chris@110:     desc.minValue = 0;
Chris@344:     desc.maxValue = 1;
Chris@316:     desc.defaultValue = int(defaultMode);
Chris@110:     desc.isQuantized = true;
Chris@110:     desc.quantizeStep = 1;
Chris@341:     desc.valueNames.push_back("Live (faster and lower latency)");
Chris@165:     desc.valueNames.push_back("Intensive (higher quality)");
Chris@161:     list.push_back(desc);
Chris@161: 
Chris@176:     desc.identifier = "instrument";
Chris@176:     desc.name = "Instrument";
Chris@161:     desc.unit = "";
Chris@271:     desc.description = "The instrument or instruments known to be present in the recording. This affects the set of instrument templates used, as well as the expected level of polyphony in the output. Using a more limited set of instruments than the default will also make the plugin run faster.\nNote that this plugin cannot isolate instruments: you can't use this setting to request notes from only one instrument in a recording with several. Instead, use this as a hint to the plugin about which instruments are actually present.";
Chris@161:     desc.minValue = 0;
Chris@162:     desc.maxValue = m_instruments.size()-1;
Chris@162:     desc.defaultValue = 0;
Chris@161:     desc.isQuantized = true;
Chris@161:     desc.quantizeStep = 1;
Chris@161:     desc.valueNames.clear();
Chris@162:     for (int i = 0; i < int(m_instruments.size()); ++i) {
Chris@162:         desc.valueNames.push_back(m_instruments[i].name);
Chris@162:     }
Chris@166:     list.push_back(desc);
Chris@161: 
Chris@166:     desc.identifier = "finetune";
Chris@166:     desc.name = "Return fine pitch estimates";
Chris@166:     desc.unit = "";
Chris@271:     desc.description = "Return pitch estimates at finer than semitone resolution. This works only in Intensive mode. Notes that appear to drift in pitch will be split up into shorter notes with individually finer pitches.";
Chris@166:     desc.minValue = 0;
Chris@166:     desc.maxValue = 1;
Chris@166:     desc.defaultValue = 0;
Chris@166:     desc.isQuantized = true;
Chris@166:     desc.quantizeStep = 1;
Chris@166:     desc.valueNames.clear();
Chris@110:     list.push_back(desc);
Chris@110: 
Chris@31:     return list;
Chris@31: }
Chris@31: 
Chris@31: float
Chris@31: Silvet::getParameter(string identifier) const
Chris@31: {
Chris@110:     if (identifier == "mode") {
Chris@297:         return (float)(int)m_mode;
Chris@166:     } else if (identifier == "finetune") {
Chris@166:         return m_fineTuning ? 1.f : 0.f;
Chris@176:     } else if (identifier == "instrument") {
Chris@162:         return m_instrument;
Chris@110:     }
Chris@31:     return 0;
Chris@31: }
Chris@31: 
Chris@31: void
Chris@31: Silvet::setParameter(string identifier, float value) 
Chris@31: {
Chris@110:     if (identifier == "mode") {
Chris@297:         m_mode = (ProcessingMode)(int)(value + 0.5);
Chris@166:     } else if (identifier == "finetune") {
Chris@166:         m_fineTuning = (value > 0.5);
Chris@176:     } else if (identifier == "instrument") {
Chris@162:         m_instrument = lrintf(value);
Chris@110:     }
Chris@31: }
Chris@31: 
Chris@31: Silvet::ProgramList
Chris@31: Silvet::getPrograms() const
Chris@31: {
Chris@31:     ProgramList list;
Chris@31:     return list;
Chris@31: }
Chris@31: 
Chris@31: string
Chris@31: Silvet::getCurrentProgram() const
Chris@31: {
Chris@31:     return ""; 
Chris@31: }
Chris@31: 
Chris@31: void
Chris@31: Silvet::selectProgram(string name)
Chris@31: {
Chris@31: }
Chris@31: 
Chris@31: Silvet::OutputList
Chris@31: Silvet::getOutputDescriptors() const
Chris@31: {
Chris@31:     OutputList list;
Chris@31: 
Chris@31:     OutputDescriptor d;
Chris@51:     d.identifier = "notes";
Chris@51:     d.name = "Note transcription";
Chris@329:     d.description = "Overall note transcription. Each note has time, duration, estimated fundamental frequency, and a synthetic MIDI velocity (1-127) estimated from the strength of the pitch in the mixture.";
Chris@41:     d.unit = "Hz";
Chris@31:     d.hasFixedBinCount = true;
Chris@31:     d.binCount = 2;
Chris@41:     d.binNames.push_back("Frequency");
Chris@31:     d.binNames.push_back("Velocity");
Chris@31:     d.hasKnownExtents = false;
Chris@31:     d.isQuantized = false;
Chris@31:     d.sampleType = OutputDescriptor::VariableSampleRate;
Chris@246:     d.sampleRate = processingSampleRate / (m_cq ? m_cq->getColumnHop() : 62);
Chris@31:     d.hasDuration = true;
Chris@32:     m_notesOutputNo = list.size();
Chris@32:     list.push_back(d);
Chris@32: 
Chris@319:     d.identifier = "onsets";
Chris@319:     d.name = "Note onsets";
Chris@323:     d.description = "Note onsets, without durations. These can be calculated sooner than complete notes, because it isn't necessary to wait for a note to finish before returning its feature. Each event has time, estimated fundamental frequency in Hz, and a synthetic MIDI velocity (1-127) estimated from the strength of the pitch in the mixture.";
Chris@319:     d.unit = "Hz";
Chris@319:     d.hasFixedBinCount = true;
Chris@319:     d.binCount = 2;
Chris@319:     d.binNames.push_back("Frequency");
Chris@319:     d.binNames.push_back("Velocity");
Chris@319:     d.hasKnownExtents = false;
Chris@319:     d.isQuantized = false;
Chris@319:     d.sampleType = OutputDescriptor::VariableSampleRate;
Chris@319:     d.sampleRate = processingSampleRate / (m_cq ? m_cq->getColumnHop() : 62);
Chris@319:     d.hasDuration = false;
Chris@319:     m_onsetsOutputNo = list.size();
Chris@319:     list.push_back(d);
Chris@319: 
Chris@336:     d.identifier = "onoffsets";
Chris@336:     d.name = "Note onsets and offsets";
Chris@336:     d.description = "Note onsets and offsets as separate events. Each onset event has time, estimated fundamental frequency in Hz, and a synthetic MIDI velocity (1-127) estimated from the strength of the pitch in the mixture. Offsets are represented in the same way but with a velocity of 0.";
Chris@336:     d.unit = "Hz";
Chris@336:     d.hasFixedBinCount = true;
Chris@336:     d.binCount = 2;
Chris@336:     d.binNames.push_back("Frequency");
Chris@336:     d.binNames.push_back("Velocity");
Chris@336:     d.hasKnownExtents = false;
Chris@336:     d.isQuantized = false;
Chris@336:     d.sampleType = OutputDescriptor::VariableSampleRate;
Chris@336:     d.sampleRate = processingSampleRate / (m_cq ? m_cq->getColumnHop() : 62);
Chris@336:     d.hasDuration = false;
Chris@336:     m_onOffsetsOutputNo = list.size();
Chris@336:     list.push_back(d);
Chris@336: 
Chris@178:     d.identifier = "timefreq";
Chris@178:     d.name = "Time-frequency distribution";
Chris@271:     d.description = "Filtered constant-Q time-frequency distribution as used as input to the expectation-maximisation algorithm.";
Chris@178:     d.unit = "";
Chris@178:     d.hasFixedBinCount = true;
Chris@298:     d.binCount = getPack(0).templateHeight;
Chris@178:     d.binNames.clear();
Chris@178:     if (m_cq) {
Chris@294:         char name[50];
Chris@298:         for (int i = 0; i < getPack(0).templateHeight; ++i) {
Chris@178:             // We have a 600-bin (10 oct 60-bin CQ) of which the
Chris@178:             // lowest-frequency 55 bins have been dropped, for a
Chris@178:             // 545-bin template. The native CQ bins go high->low
Chris@178:             // frequency though, so these are still the first 545 bins
Chris@178:             // as reported by getBinFrequency, though in reverse order
Chris@178:             float freq = m_cq->getBinFrequency
Chris@298:                 (getPack(0).templateHeight - i - 1);
Chris@178:             sprintf(name, "%.1f Hz", freq);
Chris@178:             d.binNames.push_back(name);
Chris@178:         }
Chris@178:     }
Chris@178:     d.hasKnownExtents = false;
Chris@178:     d.isQuantized = false;
Chris@178:     d.sampleType = OutputDescriptor::FixedSampleRate;
Chris@178:     d.sampleRate = m_colsPerSec;
Chris@178:     d.hasDuration = false;
Chris@178:     m_fcqOutputNo = list.size();
Chris@178:     list.push_back(d);
Chris@178: 
Chris@294:     d.identifier = "pitchactivation";
Chris@294:     d.name = "Pitch activation distribution";
Chris@294:     d.description = "Pitch activation distribution resulting from expectation-maximisation algorithm, prior to note extraction.";
Chris@294:     d.unit = "";
Chris@294:     d.hasFixedBinCount = true;
Chris@298:     d.binCount = getPack(0).templateNoteCount;
Chris@294:     d.binNames.clear();
Chris@294:     if (m_cq) {
Chris@298:         for (int i = 0; i < getPack(0).templateNoteCount; ++i) {
Chris@336:             d.binNames.push_back(getNoteName(i, 0));
Chris@294:         }
Chris@294:     }
Chris@294:     d.hasKnownExtents = false;
Chris@294:     d.isQuantized = false;
Chris@294:     d.sampleType = OutputDescriptor::FixedSampleRate;
Chris@294:     d.sampleRate = m_colsPerSec;
Chris@294:     d.hasDuration = false;
Chris@294:     m_pitchOutputNo = list.size();
Chris@294:     list.push_back(d);
Chris@294: 
Chris@309:     d.identifier = "chroma";
Chris@309:     d.name = "Pitch chroma distribution";
Chris@309:     d.description = "Pitch chroma distribution formed by wrapping the un-thresholded pitch activation distribution into a single octave of semitone bins.";
Chris@309:     d.unit = "";
Chris@309:     d.hasFixedBinCount = true;
Chris@309:     d.binCount = 12;
Chris@309:     d.binNames.clear();
Chris@309:     if (m_cq) {
Chris@309:         for (int i = 0; i < 12; ++i) {
Chris@320:             d.binNames.push_back(getChromaName(i));
Chris@309:         }
Chris@309:     }
Chris@309:     d.hasKnownExtents = false;
Chris@309:     d.isQuantized = false;
Chris@309:     d.sampleType = OutputDescriptor::FixedSampleRate;
Chris@309:     d.sampleRate = m_colsPerSec;
Chris@309:     d.hasDuration = false;
Chris@309:     m_chromaOutputNo = list.size();
Chris@309:     list.push_back(d);
Chris@309: 
Chris@302:     d.identifier = "templates";
Chris@302:     d.name = "Templates";
Chris@302:     d.description = "Constant-Q spectral templates for the selected instrument pack.";
Chris@302:     d.unit = "";
Chris@302:     d.hasFixedBinCount = true;
Chris@302:     d.binCount = getPack(0).templateHeight;
Chris@302:     d.binNames.clear();
Chris@302:     if (m_cq) {
Chris@302:         char name[50];
Chris@302:         for (int i = 0; i < getPack(0).templateHeight; ++i) {
Chris@302:             // We have a 600-bin (10 oct 60-bin CQ) of which the
Chris@302:             // lowest-frequency 55 bins have been dropped, for a
Chris@302:             // 545-bin template. The native CQ bins go high->low
Chris@302:             // frequency though, so these are still the first 545 bins
Chris@302:             // as reported by getBinFrequency, though in reverse order
Chris@302:             float freq = m_cq->getBinFrequency
Chris@302:                 (getPack(0).templateHeight - i - 1);
Chris@302:             sprintf(name, "%.1f Hz", freq);
Chris@302:             d.binNames.push_back(name);
Chris@302:         }
Chris@302:     }
Chris@302:     d.hasKnownExtents = false;
Chris@302:     d.isQuantized = false;
Chris@302:     d.sampleType = OutputDescriptor::FixedSampleRate;
Chris@302:     d.sampleRate = m_colsPerSec;
Chris@302:     d.hasDuration = false;
Chris@302:     m_templateOutputNo = list.size();
Chris@302:     list.push_back(d);
Chris@302: 
Chris@31:     return list;
Chris@31: }
Chris@31: 
Chris@38: std::string
Chris@320: Silvet::getChromaName(int pitch) const
Chris@38: {
Chris@38:     static const char *names[] = {
Chris@38:         "A", "A#", "B", "C", "C#", "D", "D#", "E", "F", "F#", "G", "G#"
Chris@38:     };
Chris@38: 
Chris@309:     return names[pitch];
Chris@309: }
Chris@309:     
Chris@309: std::string
Chris@336: Silvet::getNoteName(int note, int shift) const
Chris@309: {
Chris@320:     string n = getChromaName(note % 12);
Chris@38: 
Chris@175:     int oct = (note + 9) / 12; 
Chris@38:     
Chris@175:     char buf[30];
Chris@175: 
Chris@175:     float pshift = 0.f;
Chris@336:     int shiftCount = getShiftCount();
Chris@175:     if (shiftCount > 1) {
Chris@320:         // see getNoteFrequency below
Chris@175:         pshift = 
Chris@175:             float((shiftCount - shift) - int(shiftCount / 2) - 1) / shiftCount;
Chris@175:     }
Chris@175: 
Chris@175:     if (pshift > 0.f) {
Chris@309:         sprintf(buf, "%s%d+%dc", n.c_str(), oct, int(round(pshift * 100)));
Chris@175:     } else if (pshift < 0.f) {
Chris@309:         sprintf(buf, "%s%d-%dc", n.c_str(), oct, int(round((-pshift) * 100)));
Chris@175:     } else {
Chris@309:         sprintf(buf, "%s%d", n.c_str(), oct);
Chris@175:     }
Chris@38: 
Chris@38:     return buf;
Chris@38: }
Chris@38: 
Chris@41: float
Chris@336: Silvet::getNoteFrequency(int note, int shift) const
Chris@41: {
Chris@169:     // Convert shift number to a pitch shift. The given shift number
Chris@169:     // is an offset into the template array, which starts with some
Chris@169:     // zeros, followed by the template, then some trailing zeros.
Chris@169:     // 
Chris@169:     // Example: if we have templateMaxShift == 2 and thus shiftCount
Chris@169:     // == 5, then the number will be in the range 0-4 and the template
Chris@169:     // will have 2 zeros at either end. Thus number 2 represents the
Chris@169:     // template "as recorded", for a pitch shift of 0; smaller indices
Chris@169:     // represent moving the template *up* in pitch (by introducing
Chris@169:     // zeros at the start, which is the low-frequency end), for a
Chris@169:     // positive pitch shift; and higher values represent moving it
Chris@169:     // down in pitch, for a negative pitch shift.
Chris@169: 
Chris@175:     float pshift = 0.f;
Chris@336:     int shiftCount = getShiftCount();
Chris@175:     if (shiftCount > 1) {
Chris@175:         pshift = 
Chris@175:             float((shiftCount - shift) - int(shiftCount / 2) - 1) / shiftCount;
Chris@175:     }
Chris@169: 
Chris@301:     float freq = float(27.5 * pow(2.0, (note + pshift) / 12.0));
Chris@301: 
Chris@303: //    cerr << "note = " << note << ", shift = " << shift << ", shiftCount = "
Chris@303: //         << shiftCount << ", obtained freq = " << freq << endl;
Chris@301:     
Chris@301:     return freq;
Chris@41: }
Chris@41: 
Chris@31: bool
Chris@31: Silvet::initialise(size_t channels, size_t stepSize, size_t blockSize)
Chris@31: {
Chris@272:     if (m_inputSampleRate < minInputSampleRate ||
Chris@272:         m_inputSampleRate > maxInputSampleRate) {
Chris@272: 	cerr << "Silvet::initialise: Unsupported input sample rate "
Chris@272:              << m_inputSampleRate << " (supported min " << minInputSampleRate
Chris@272:              << ", max " << maxInputSampleRate << ")" << endl;
Chris@272:         return false;
Chris@272:     }
Chris@272: 
Chris@31:     if (channels < getMinChannelCount() ||
Chris@272: 	channels > getMaxChannelCount()) {
Chris@272: 	cerr << "Silvet::initialise: Unsupported channel count " << channels
Chris@272:              << " (supported min " << getMinChannelCount() << ", max "
Chris@272:              << getMaxChannelCount() << ")" << endl;
Chris@272:         return false;
Chris@272:     }
Chris@31: 
Chris@31:     if (stepSize != blockSize) {
Chris@31: 	cerr << "Silvet::initialise: Step size must be the same as block size ("
Chris@31: 	     << stepSize << " != " << blockSize << ")" << endl;
Chris@31: 	return false;
Chris@31:     }
Chris@31: 
Chris@31:     m_blockSize = blockSize;
Chris@31: 
Chris@31:     reset();
Chris@31: 
Chris@31:     return true;
Chris@31: }
Chris@31: 
Chris@31: void
Chris@31: Silvet::reset()
Chris@31: {
Chris@31:     delete m_resampler;
Chris@246:     delete m_flattener;
Chris@31:     delete m_cq;
Chris@31: 
Chris@31:     if (m_inputSampleRate != processingSampleRate) {
Chris@31: 	m_resampler = new Resampler(m_inputSampleRate, processingSampleRate);
Chris@31:     } else {
Chris@31: 	m_resampler = 0;
Chris@31:     }
Chris@31: 
Chris@246:     m_flattener = new FlattenDynamics(m_inputSampleRate); // before resampling
Chris@246:     m_flattener->reset();
Chris@246: 
Chris@301:     // this happens to be processingSampleRate / 3, and is the top
Chris@301:     // freq used for the EM templates:
Chris@301:     double maxFreq = 14700;
Chris@301: 
Chris@301:     if (m_mode == LiveMode) {
Chris@301:         // We only have 12 bpo rather than 60, so we need the top bin
Chris@301:         // to be the middle one of the top 5, i.e. 2/5 of a semitone
Chris@301:         // lower than 14700
Chris@301:         maxFreq *= powf(2.0, -1.0 / 30.0);
Chris@301:     }
Chris@301:     
Chris@173:     double minFreq = 27.5;
Chris@173: 
Chris@341:     if (m_mode == LiveMode) {
Chris@173:         // We don't actually return any notes from the bottom octave,
Chris@173:         // so we can just pad with zeros
Chris@173:         minFreq *= 2;
Chris@173:     }
Chris@173: 
Chris@298:     int bpo = 12 *
Chris@298:         (m_mode == LiveMode ? binsPerSemitoneLive : binsPerSemitoneNormal);
Chris@301: 
Chris@154:     CQParameters params(processingSampleRate,
Chris@173:                         minFreq, 
Chris@303:                         maxFreq,
Chris@298:                         bpo);
Chris@154: 
Chris@325:     params.q = 0.8;
Chris@325:     params.atomHopFactor = (m_mode == LiveMode ? 1.0 : 0.3);
Chris@154:     params.threshold = 0.0005;
Chris@317:     params.decimator =
Chris@317:         (m_mode == LiveMode ?
Chris@317:          CQParameters::FasterDecimator : CQParameters::BetterDecimator);
Chris@172:     params.window = CQParameters::Hann;
Chris@154: 
Chris@154:     m_cq = new CQSpectrogram(params, CQSpectrogram::InterpolateLinear);
Chris@31: 
Chris@303: //    cerr << "CQ bins = " << m_cq->getTotalBins() << endl;
Chris@303: //    cerr << "CQ min freq = " << m_cq->getMinFrequency() << " (and for confirmation, freq of bin 0 = " << m_cq->getBinFrequency(0) << ")" << endl;
Chris@297:     
Chris@341:     m_colsPerSec = 50;
Chris@165: 
Chris@41:     for (int i = 0; i < (int)m_postFilter.size(); ++i) {
Chris@41:         delete m_postFilter[i];
Chris@41:     }
Chris@41:     m_postFilter.clear();
Chris@303:     int postFilterLength = 3;
Chris@298:     for (int i = 0; i < getPack(0).templateNoteCount; ++i) {
Chris@303:         m_postFilter.push_back(new MedianFilter<double>(postFilterLength));
Chris@41:     }
Chris@41:     m_pianoRoll.clear();
Chris@246:     m_inputGains.clear();
Chris@32:     m_columnCount = 0;
Chris@272:     m_resampledCount = 0;
Chris@40:     m_startTime = RealTime::zeroTime;
Chris@313:     m_haveStartTime = false;
Chris@31: }
Chris@31: 
Chris@31: Silvet::FeatureSet
Chris@31: Silvet::process(const float *const *inputBuffers, Vamp::RealTime timestamp)
Chris@31: {
Chris@302:     FeatureSet fs;
Chris@302:     
Chris@313:     if (!m_haveStartTime) {
Chris@314: 
Chris@40:         m_startTime = timestamp;
Chris@313:         m_haveStartTime = true;
Chris@314: 
Chris@302:         insertTemplateFeatures(fs);
Chris@40:     }
Chris@246: 
Chris@246:     vector<float> flattened(m_blockSize);
Chris@246:     float gain = 1.f;
Chris@246:     m_flattener->connectInputPort
Chris@246:         (FlattenDynamics::AudioInputPort, inputBuffers[0]);
Chris@246:     m_flattener->connectOutputPort
Chris@246:         (FlattenDynamics::AudioOutputPort, &flattened[0]);
Chris@246:     m_flattener->connectOutputPort
Chris@246:         (FlattenDynamics::GainOutputPort, &gain);
Chris@246:     m_flattener->process(m_blockSize);
Chris@246: 
Chris@252:     m_inputGains[timestamp] = gain;
Chris@40:     
Chris@31:     vector<double> data;
Chris@40:     for (int i = 0; i < m_blockSize; ++i) {
Chris@246:         double d = flattened[i];
Chris@235:         data.push_back(d);
Chris@40:     }
Chris@31: 
Chris@31:     if (m_resampler) {
Chris@272: 
Chris@31: 	data = m_resampler->process(data.data(), data.size());
Chris@272: 
Chris@272:         int hadCount = m_resampledCount;
Chris@272:         m_resampledCount += data.size();
Chris@272: 
Chris@272:         int resamplerLatency = m_resampler->getLatency();
Chris@272: 
Chris@272:         if (hadCount < resamplerLatency) {
Chris@272:             int stillToDrop = resamplerLatency - hadCount;
Chris@272:             if (stillToDrop >= int(data.size())) {
Chris@302:                 return fs;
Chris@272:             } else {
Chris@272:                 data = vector<double>(data.begin() + stillToDrop, data.end());
Chris@272:             }
Chris@272:         }
Chris@31:     }
Chris@272: 
Chris@32:     Grid cqout = m_cq->process(data);
Chris@302:     transcribe(cqout, fs);
Chris@51:     return fs;
Chris@34: }
Chris@34: 
Chris@34: Silvet::FeatureSet
Chris@34: Silvet::getRemainingFeatures()
Chris@34: {
Chris@145:     Grid cqout = m_cq->getRemainingOutput();
Chris@302:     FeatureSet fs;
Chris@336: 
Chris@302:     if (m_columnCount == 0) {
Chris@302:         // process() was never called, but we still want these
Chris@302:         insertTemplateFeatures(fs);
Chris@302:     } else {
Chris@336: 
Chris@336:         // Complete the transcription
Chris@336: 
Chris@302:         transcribe(cqout, fs);
Chris@336: 
Chris@336:         // And make sure any extant playing notes are finished and returned
Chris@336: 
Chris@336:         m_pianoRoll.push_back({});
Chris@336: 
Chris@336:         auto events = noteTrack();
Chris@336:     
Chris@336:         for (const auto &f : events.notes) {
Chris@336:             fs[m_notesOutputNo].push_back(f);
Chris@336:         }
Chris@336:     
Chris@336:         for (const auto &f : events.onsets) {
Chris@336:             fs[m_onsetsOutputNo].push_back(f);
Chris@336:         }
Chris@336:     
Chris@336:         for (const auto &f : events.onOffsets) {
Chris@336:             fs[m_onOffsetsOutputNo].push_back(f);
Chris@336:         }
Chris@302:     }
Chris@336: 
Chris@51:     return fs;
Chris@34: }
Chris@34: 
Chris@302: void
Chris@302: Silvet::insertTemplateFeatures(FeatureSet &fs)
Chris@302: {
Chris@302:     const InstrumentPack &pack = getPack(m_instrument);
Chris@302:     for (int i = 0; i < int(pack.templates.size()) * pack.templateNoteCount; ++i) {
Chris@302:         RealTime timestamp = RealTime::fromSeconds(double(i) / m_colsPerSec);
Chris@302:         Feature f;
Chris@302:         char buffer[50];
Chris@302:         sprintf(buffer, "Note %d", i + 1);
Chris@302:         f.label = buffer;
Chris@302:         f.hasTimestamp = true;
Chris@302:         f.timestamp = timestamp;
Chris@302:         f.values = pack.templates[i / pack.templateNoteCount]
Chris@302:             .data[i % pack.templateNoteCount];
Chris@302:         fs[m_templateOutputNo].push_back(f);
Chris@302:     }
Chris@302: }        
Chris@302: 
Chris@336: int
Chris@336: Silvet::getShiftCount() const
Chris@336: {
Chris@336:     bool wantShifts = (m_mode == HighQualityMode) && m_fineTuning;
Chris@336:     int shiftCount = 1;
Chris@336:     if (wantShifts) {
Chris@336:         const InstrumentPack &pack(getPack(m_instrument));
Chris@336:         shiftCount = pack.templateMaxShift * 2 + 1;
Chris@336:     }
Chris@336:     return shiftCount;
Chris@336: }
Chris@336: 
Chris@302: void
Chris@302: Silvet::transcribe(const Grid &cqout, Silvet::FeatureSet &fs)
Chris@34: {
Chris@32:     Grid filtered = preProcess(cqout);
Chris@31: 
Chris@302:     if (filtered.empty()) return;
Chris@170:     
Chris@298:     const InstrumentPack &pack(getPack(m_instrument));
Chris@104: 
Chris@325:     int width = filtered.size();
Chris@325: 
Chris@325:     double silenceThreshold = 0.01;
Chris@325:     
Chris@325:     for (int i = 0; i < width; ++i) {
Chris@325: 
Chris@325:         RealTime timestamp = getColumnTimestamp(m_pianoRoll.size() - 1 + i);
Chris@325:         float inputGain = getInputGainAt(timestamp);
Chris@325: 
Chris@178:         Feature f;
Chris@325:         double rms = 0.0;
Chris@325: 
Chris@178:         for (int j = 0; j < pack.templateHeight; ++j) {
Chris@325:             double v = filtered[i][j];
Chris@325:             rms += v * v;
Chris@325:             f.values.push_back(float(v));
Chris@178:         }
Chris@325: 
Chris@325:         rms = sqrt(rms / pack.templateHeight);
Chris@325:         if (rms / inputGain < silenceThreshold) {
Chris@325:             filtered[i].clear();
Chris@325:         }
Chris@325:         
Chris@178:         fs[m_fcqOutputNo].push_back(f);
Chris@178:     }
Chris@325:     
Chris@311:     Grid localPitches(width);
Chris@170: 
Chris@336:     int shiftCount = getShiftCount();
Chris@336:     bool wantShifts = (shiftCount > 1);
Chris@170: 
Chris@170:     vector<vector<int> > localBestShifts;
Chris@170:     if (wantShifts) {
Chris@311:         localBestShifts = vector<vector<int> >(width);
Chris@170:     }
Chris@170: 
Chris@356:     int emThreadCount = 1;
Chris@356: 
Chris@356: #if (defined(MAX_EM_THREADS) && (MAX_EM_THREADS > 1))
Chris@356:     emThreadCount = MAX_EM_THREADS;
Chris@356: 
Chris@352:     if (emThreadCount > int(std::thread::hardware_concurrency())) {
Chris@352:         emThreadCount = std::thread::hardware_concurrency();
Chris@352:     }
Chris@317:     if (m_mode == LiveMode && pack.templates.size() == 1) {
Chris@317:         // The EM step is probably not slow enough to merit it
Chris@317:         emThreadCount = 1;
Chris@317:     }
Chris@317: 
Chris@317:     if (emThreadCount > 1) {
Chris@317:         for (int i = 0; i < width; ) {
Chris@317:             typedef future<pair<vector<double>, vector<int>>> EMFuture;
Chris@317:             vector<EMFuture> results;
Chris@317:             for (int j = 0; j < emThreadCount && i + j < width; ++j) {
Chris@352:                 const vector<double> &column = filtered.at(i + j);
Chris@317:                 results.push_back
Chris@317:                     (async(std::launch::async,
Chris@352:                            [&]() { return applyEM(pack, column); }));
Chris@317:             }
Chris@317:             for (int j = 0; j < emThreadCount && i + j < width; ++j) {
Chris@317:                 auto out = results[j].get();
Chris@317:                 localPitches[i+j] = out.first;
Chris@317:                 if (wantShifts) localBestShifts[i+j] = out.second;
Chris@317:             }
Chris@317:             i += emThreadCount;
Chris@312:         }
Chris@123:     }
Chris@312: #endif
Chris@317: 
Chris@317:     if (emThreadCount == 1) {
Chris@317:         for (int i = 0; i < width; ++i) {
Chris@336:             auto out = applyEM(pack, filtered.at(i));
Chris@317:             localPitches[i] = out.first;
Chris@317:             if (wantShifts) localBestShifts[i] = out.second;
Chris@317:         }
Chris@317:     }
Chris@305:         
Chris@166:     for (int i = 0; i < width; ++i) {
Chris@37: 
Chris@321:         vector<double> filtered;
Chris@321: 
Chris@321:         for (int j = 0; j < pack.templateNoteCount; ++j) {
Chris@321:             m_postFilter[j]->push(localPitches[i][j]);
Chris@321:             filtered.push_back(m_postFilter[j]->get());
Chris@321:         }
Chris@294: 
Chris@309:         RealTime timestamp = getColumnTimestamp(m_pianoRoll.size() - 1);
Chris@309:         float inputGain = getInputGainAt(timestamp);
Chris@309: 
Chris@294:         Feature f;
Chris@294:         for (int j = 0; j < (int)filtered.size(); ++j) {
Chris@309:             float v = filtered[j];
Chris@294:             if (v < pack.levelThreshold) v = 0.f;
Chris@309:             f.values.push_back(v / inputGain);
Chris@294:         }
Chris@294:         fs[m_pitchOutputNo].push_back(f);
Chris@309: 
Chris@309:         f.values.clear();
Chris@309:         f.values.resize(12);
Chris@309:         for (int j = 0; j < (int)filtered.size(); ++j) {
Chris@309:             f.values[j % 12] += filtered[j] / inputGain;
Chris@309:         }
Chris@309:         fs[m_chromaOutputNo].push_back(f);
Chris@38: 
Chris@321:         // This pushes the up-to-max-polyphony activation column to
Chris@321:         // m_pianoRoll
Chris@336:         postProcess(filtered, localBestShifts[i]);
Chris@321: 
Chris@336:         auto events = noteTrack();
Chris@319: 
Chris@336:         for (const auto &f : events.notes) {
Chris@336:             fs[m_notesOutputNo].push_back(f);
Chris@40:         }
Chris@319: 
Chris@336:         for (const auto &f : events.onsets) {
Chris@336:             fs[m_onsetsOutputNo].push_back(f);
Chris@336:         }
Chris@336: 
Chris@336:         for (const auto &f : events.onOffsets) {
Chris@336:             fs[m_onOffsetsOutputNo].push_back(f);
Chris@319:         }
Chris@34:     }
Chris@31: }
Chris@31: 
Chris@311: pair<vector<double>, vector<int> >
Chris@311: Silvet::applyEM(const InstrumentPack &pack,
Chris@336:                 const vector<double> &column)
Chris@311: {
Chris@311:     double columnThreshold = 1e-5;
Chris@311:     
Chris@314:     if (m_mode == LiveMode) {
Chris@325:         columnThreshold /= 15;
Chris@314:     }
Chris@314:     
Chris@311:     vector<double> pitches(pack.templateNoteCount, 0.0);
Chris@311:     vector<int> bestShifts;
Chris@325: 
Chris@325:     if (column.empty()) return { pitches, bestShifts };
Chris@311:     
Chris@311:     double sum = 0.0;
Chris@311:     for (int j = 0; j < pack.templateHeight; ++j) {
Chris@311:         sum += column.at(j);
Chris@311:     }
Chris@311:     if (sum < columnThreshold) return { pitches, bestShifts };
Chris@311: 
Chris@314:     EM em(&pack, m_mode == HighQualityMode);
Chris@311: 
Chris@311:     em.setPitchSparsity(pack.pitchSparsity);
Chris@311:     em.setSourceSparsity(pack.sourceSparsity);
Chris@311: 
Chris@314:     int iterations = (m_mode == HighQualityMode ? 20 : 10);
Chris@311: 
Chris@311:     for (int j = 0; j < iterations; ++j) {
Chris@311:         em.iterate(column.data());
Chris@311:     }
Chris@311: 
Chris@311:     const float *pitchDist = em.getPitchDistribution();
Chris@311:     const float *const *shiftDist = em.getShifts();
Chris@311: 
Chris@336:     int shiftCount = getShiftCount();
Chris@311:     
Chris@311:     for (int j = 0; j < pack.templateNoteCount; ++j) {
Chris@311: 
Chris@311:         pitches[j] = pitchDist[j] * sum;
Chris@311: 
Chris@311:         int bestShift = 0;
Chris@311:         float bestShiftValue = 0.0;
Chris@336:         if (shiftCount > 1) {
Chris@311:             for (int k = 0; k < shiftCount; ++k) {
Chris@311:                 float value = shiftDist[k][j];
Chris@311:                 if (k == 0 || value > bestShiftValue) {
Chris@311:                     bestShiftValue = value;
Chris@311:                     bestShift = k;
Chris@311:                 }
Chris@311:             }
Chris@311:             bestShifts.push_back(bestShift);
Chris@311:         }                
Chris@311:     }
Chris@311: 
Chris@311:     return { pitches, bestShifts };
Chris@311: }
Chris@311: 
Chris@32: Silvet::Grid
Chris@32: Silvet::preProcess(const Grid &in)
Chris@32: {
Chris@32:     int width = in.size();
Chris@32: 
Chris@165:     int spacing = processingSampleRate / m_colsPerSec;
Chris@32: 
Chris@165:     // need to be careful that col spacing is an integer number of samples!
Chris@165:     assert(spacing * m_colsPerSec == processingSampleRate);
Chris@32: 
Chris@32:     Grid out;
Chris@32: 
Chris@58:     // We count the CQ latency in terms of processing hops, but
Chris@58:     // actually it probably isn't an exact number of hops so this
Chris@58:     // isn't quite accurate. But the small constant offset is
Chris@165:     // practically irrelevant compared to the jitter from the frame
Chris@165:     // size we reduce to in a moment
Chris@33:     int latentColumns = m_cq->getLatency() / m_cq->getColumnHop();
Chris@33: 
Chris@298:     const InstrumentPack &pack(getPack(m_instrument));
Chris@176: 
Chris@32:     for (int i = 0; i < width; ++i) {
Chris@32: 
Chris@33:         if (m_columnCount < latentColumns) {
Chris@33:             ++m_columnCount;
Chris@33:             continue;
Chris@33:         }
Chris@33: 
Chris@32:         int prevSampleNo = (m_columnCount - 1) * m_cq->getColumnHop();
Chris@32:         int sampleNo = m_columnCount * m_cq->getColumnHop();
Chris@32: 
Chris@32:         bool select = (sampleNo / spacing != prevSampleNo / spacing);
Chris@32: 
Chris@32:         if (select) {
Chris@32:             vector<double> inCol = in[i];
Chris@176:             vector<double> outCol(pack.templateHeight);
Chris@32: 
Chris@178:             // In HQ mode, the CQ returns 600 bins and we ignore the
Chris@298:             // lowest 55 of them (assuming binsPerSemitone == 5).
Chris@178:             // 
Chris@341:             // In live mode the CQ is an octave shorter, returning 540
Chris@341:             // bins or equivalent, so we instead pad them with an
Chris@341:             // additional 5 or equivalent zeros.
Chris@178:             // 
Chris@178:             // We also need to reverse the column as we go, since the
Chris@178:             // raw CQ has the high frequencies first and we need it
Chris@178:             // the other way around.
Chris@32: 
Chris@298:             int bps = (m_mode == LiveMode ?
Chris@298:                        binsPerSemitoneLive : binsPerSemitoneNormal);
Chris@298:             
Chris@297:             if (m_mode == HighQualityMode) {
Chris@178:                 for (int j = 0; j < pack.templateHeight; ++j) {
Chris@298:                     int ix = inCol.size() - j - (11 * bps);
Chris@178:                     outCol[j] = inCol[ix];
Chris@178:                 }
Chris@178:             } else {
Chris@298:                 for (int j = 0; j < bps; ++j) {
Chris@178:                     outCol[j] = 0.0;
Chris@178:                 }
Chris@298:                 for (int j = bps; j < pack.templateHeight; ++j) {
Chris@298:                     int ix = inCol.size() - j + (bps-1);
Chris@178:                     outCol[j] = inCol[ix];
Chris@178:                 }
Chris@46:             }
Chris@32: 
Chris@46:             vector<double> noiseLevel1 = 
Chris@298:                 MedianFilter<double>::filter(8 * bps, outCol);
Chris@176:             for (int j = 0; j < pack.templateHeight; ++j) {
Chris@46:                 noiseLevel1[j] = std::min(outCol[j], noiseLevel1[j]);
Chris@46:             }
Chris@32: 
Chris@46:             vector<double> noiseLevel2 = 
Chris@298:                 MedianFilter<double>::filter(8 * bps, noiseLevel1);
Chris@176:             for (int j = 0; j < pack.templateHeight; ++j) {
Chris@46:                 outCol[j] = std::max(outCol[j] - noiseLevel2[j], 0.0);
Chris@32:             }
Chris@32: 
Chris@165:             out.push_back(outCol);
Chris@32:         }
Chris@32: 
Chris@32:         ++m_columnCount;
Chris@32:     }
Chris@32: 
Chris@32:     return out;
Chris@32: }
Chris@32:     
Chris@321: void
Chris@170: Silvet::postProcess(const vector<double> &pitches,
Chris@336:                     const vector<int> &bestShifts)
Chris@166: {
Chris@298:     const InstrumentPack &pack(getPack(m_instrument));
Chris@176: 
Chris@41:     // Threshold for level and reduce number of candidate pitches
Chris@41: 
Chris@41:     typedef std::multimap<double, int> ValueIndexMap;
Chris@41: 
Chris@41:     ValueIndexMap strengths;
Chris@166: 
Chris@176:     for (int j = 0; j < pack.templateNoteCount; ++j) {
Chris@321: 
Chris@321:         double strength = pitches[j];
Chris@183:         if (strength < pack.levelThreshold) continue;
Chris@321:         
Chris@321:         // In live mode with only a 12-bpo CQ, we are very likely to
Chris@321:         // get clusters of two or three high scores at a time for
Chris@321:         // neighbouring semitones. Eliminate these by picking only the
Chris@325:         // peaks (except that we never eliminate a note that has
Chris@325:         // already been established as currently playing). This means
Chris@325:         // we can't recognise actual semitone chords if they ever
Chris@325:         // appear, but it's not as if live mode is good enough for
Chris@325:         // that to be a big deal anyway.
Chris@321:         if (m_mode == LiveMode) {
Chris@325:             if (m_current.find(j) == m_current.end() &&
Chris@325:                 (j == 0 ||
Chris@325:                  j + 1 == pack.templateNoteCount ||
Chris@325:                  pitches[j] < pitches[j-1] ||
Chris@325:                  pitches[j] < pitches[j+1])) {
Chris@325:                 // not a peak or a currently-playing note: skip it
Chris@321:                 continue;
Chris@321:             }
Chris@321:         }
Chris@323: 
Chris@168:         strengths.insert(ValueIndexMap::value_type(strength, j));
Chris@168:     }
Chris@166: 
Chris@168:     ValueIndexMap::const_iterator si = strengths.end();
Chris@167: 
Chris@168:     map<int, double> active;
Chris@168:     map<int, int> activeShifts;
Chris@168: 
Chris@336:     int shiftCount = getShiftCount();
Chris@336:     
Chris@183:     while (int(active.size()) < pack.maxPolyphony && si != strengths.begin()) {
Chris@168: 
Chris@168:         --si;
Chris@168: 
Chris@168:         double strength = si->first;
Chris@168:         int j = si->second;
Chris@168: 
Chris@168:         active[j] = strength;
Chris@168: 
Chris@336:         if (shiftCount > 1) {
Chris@364:             if (!bestShifts.empty()) {
Chris@364:                 activeShifts[j] = bestShifts[j];
Chris@364:             } else {
Chris@364:                 // can happen if column was below threshold
Chris@364:                 activeShifts[j] = 0;
Chris@364:             }
Chris@167:         }
Chris@41:     }
Chris@41: 
Chris@168:     m_pianoRoll.push_back(active);
Chris@170: 
Chris@336:     if (shiftCount > 1) {
Chris@168:         m_pianoRollShifts.push_back(activeShifts);
Chris@41:     }
Chris@294: 
Chris@321:     return;
Chris@166: }
Chris@166: 
Chris@336: Silvet::FeatureChunk
Chris@336: Silvet::noteTrack()
Chris@166: {        
Chris@41:     // Minimum duration pruning, and conversion to notes. We can only
Chris@41:     // report notes that have just ended (i.e. that are absent in the
Chris@168:     // latest active set but present in the prior set in the piano
Chris@41:     // roll) -- any notes that ended earlier will have been reported
Chris@41:     // already, and if they haven't ended, we don't know their
Chris@41:     // duration.
Chris@41: 
Chris@168:     int width = m_pianoRoll.size() - 1;
Chris@168: 
Chris@168:     const map<int, double> &active = m_pianoRoll[width];
Chris@41: 
Chris@165:     double columnDuration = 1.0 / m_colsPerSec;
Chris@165: 
Chris@165:     // only keep notes >= 100ms or thereabouts
Chris@323:     double durationThrSec = 0.1;
Chris@323:     int durationThreshold = floor(durationThrSec / columnDuration); // in cols
Chris@165:     if (durationThreshold < 1) durationThreshold = 1;
Chris@41: 
Chris@336:     FeatureList noteFeatures, onsetFeatures, onOffsetFeatures;
Chris@41: 
Chris@41:     if (width < durationThreshold + 1) {
Chris@336:         return { noteFeatures, onsetFeatures, onOffsetFeatures };
Chris@41:     }
Chris@357: 
Chris@357:     // Make a copy of the latest column. We need a copy because it is
Chris@357:     // possible we may erase from the "live" column within the loop.
Chris@357:     map<int, double> latest = m_pianoRoll[width-1];
Chris@41:     
Chris@357:     for (const auto &ni: latest) {
Chris@41: 
Chris@357:         int note = ni.first;
Chris@41: 
Chris@41:         int end = width;
Chris@41:         int start = end-1;
Chris@41: 
Chris@41:         while (m_pianoRoll[start].find(note) != m_pianoRoll[start].end()) {
Chris@41:             --start;
Chris@41:         }
Chris@41:         ++start;
Chris@41: 
Chris@319:         int duration = end - start;
Chris@319:             
Chris@319:         if (duration < durationThreshold) {
Chris@41:             continue;
Chris@41:         }
Chris@41: 
Chris@319:         if (duration == durationThreshold) {
Chris@325:             m_current.insert(note);
Chris@336:             emitOnset(start, note, onsetFeatures);
Chris@336:             emitOnset(start, note, onOffsetFeatures);
Chris@319:         }            
Chris@319:         
Chris@319:         if (active.find(note) == active.end()) {
Chris@319:             // the note was playing but just ended
Chris@325:             m_current.erase(note);
Chris@343:             emitNoteAndOffset(start, end, note, noteFeatures, onOffsetFeatures);
Chris@334:         } else { // still playing
Chris@334:             // repeated note detection: if level is greater than this
Chris@334:             // multiple of its previous value, then we end the note and
Chris@334:             // restart it with the same pitch
Chris@334:             double restartFactor = 1.5;
Chris@334:             if (duration >= durationThreshold * 2 &&
Chris@334:                 (active.find(note)->second >
Chris@334:                  restartFactor * m_pianoRoll[width-1][note])) {
Chris@334:                 m_current.erase(note);
Chris@343:                 emitNoteAndOffset(start, end-1, note, noteFeatures, onOffsetFeatures);
Chris@334:                 // and remove this so that we start counting the new
Chris@357:                 // note's duration from the current position. (This
Chris@357:                 // erase is why we needed to copy this column at the
Chris@357:                 // top of the loop.)
Chris@334:                 m_pianoRoll[width-1].erase(note);
Chris@334:             }
Chris@319:         }
Chris@41:     }
Chris@41: 
Chris@62: //    cerr << "returning " << noteFeatures.size() << " complete note(s) " << endl;
Chris@41: 
Chris@336:     return { noteFeatures, onsetFeatures, onOffsetFeatures };
Chris@41: }
Chris@41: 
Chris@169: void
Chris@343: Silvet::emitNoteAndOffset(int start, int end, int note,
Chris@343:                           FeatureList &noteFeatures,
Chris@343:                           FeatureList &onOffsetFeatures)
Chris@169: {
Chris@343:     // Emit the complete note-event feature, and its offset. We have
Chris@343:     // already emitted the note onset when it started -- that process
Chris@343:     // is separated out in order to get a faster response during live
Chris@343:     // tracking. However, if the note shift changes within the note
Chris@343:     // (which can happen only if we have fine-tuning switched on), we
Chris@343:     // emit an offset and then a new onset with the new shift.
Chris@343:     
Chris@169:     int partStart = start;
Chris@169:     int partShift = 0;
Chris@320:     double partStrength = 0;
Chris@169: 
Chris@343:     // NB this *must* be less than durationThreshold above
Chris@252:     int partThreshold = floor(0.05 * m_colsPerSec);
Chris@169: 
Chris@169:     for (int i = start; i != end; ++i) {
Chris@169:         
Chris@169:         double strength = m_pianoRoll[i][note];
Chris@169: 
Chris@169:         int shift = 0;
Chris@169: 
Chris@336:         if (getShiftCount() > 1) {
Chris@169: 
Chris@169:             shift = m_pianoRollShifts[i][note];
Chris@169: 
Chris@169:             if (i == partStart) {
Chris@169:                 partShift = shift;
Chris@169:             }
Chris@169: 
Chris@169:             if (i > partStart + partThreshold && shift != partShift) {
Chris@169: 
Chris@169:                 // pitch has changed, emit an intermediate note
Chris@252:                 noteFeatures.push_back(makeNoteFeature(partStart,
Chris@252:                                                        i,
Chris@252:                                                        note,
Chris@252:                                                        partShift,
Chris@320:                                                        partStrength));
Chris@343: 
Chris@343:                 onOffsetFeatures.push_back(makeOffsetFeature(i,
Chris@343:                                                              note,
Chris@343:                                                              partShift));
Chris@343:                 
Chris@169:                 partStart = i;
Chris@169:                 partShift = shift;
Chris@343: 
Chris@343:                 onOffsetFeatures.push_back(makeOnsetFeature(i,
Chris@343:                                                             note,
Chris@343:                                                             partShift,
Chris@343:                                                             partStrength));
Chris@343: 
Chris@320:                 partStrength = 0;
Chris@169:             }
Chris@169:         }
Chris@169: 
Chris@320:         if (strength > partStrength) {
Chris@320:             partStrength = strength;
Chris@169:         }
Chris@169:     }
Chris@169: 
Chris@169:     if (end >= partStart + partThreshold) {
Chris@343: 
Chris@252:         noteFeatures.push_back(makeNoteFeature(partStart,
Chris@252:                                                end,
Chris@252:                                                note,
Chris@252:                                                partShift,
Chris@320:                                                partStrength));
Chris@343: 
Chris@343:         onOffsetFeatures.push_back(makeOffsetFeature(end,
Chris@343:                                                      note,
Chris@343:                                                      partShift));
Chris@349: 
Chris@349:     } else if (partStart > start) {
Chris@349: 
Chris@349:         // we have emitted an onset for this, so must add an offset
Chris@349:         onOffsetFeatures.push_back(makeOffsetFeature(end,
Chris@349:                                                      note,
Chris@349:                                                      partShift));
Chris@169:     }
Chris@169: }
Chris@252: 
Chris@319: void
Chris@336: Silvet::emitOnset(int start, int note, FeatureList &onOffsetFeatures)
Chris@319: {
Chris@319:     int len = int(m_pianoRoll.size());
Chris@320: 
Chris@320:     double onsetStrength = 0;
Chris@319: 
Chris@319:     int shift = 0;
Chris@336:     if (getShiftCount() > 1) {
Chris@319:         shift = m_pianoRollShifts[start][note];
Chris@319:     }
Chris@319:     
Chris@319:     for (int i = start; i < len; ++i) {
Chris@319:         double strength = m_pianoRoll[i][note];
Chris@320:         if (strength > onsetStrength) {
Chris@320:             onsetStrength = strength;
Chris@319:         }
Chris@319:     }
Chris@319: 
Chris@336:     if (onsetStrength == 0) return;
Chris@336:     
Chris@336:     onOffsetFeatures.push_back(makeOnsetFeature(start,
Chris@336:                                                 note,
Chris@336:                                                 shift,
Chris@336:                                                 onsetStrength));
Chris@336: }
Chris@336: 
Chris@309: RealTime
Chris@309: Silvet::getColumnTimestamp(int column)
Chris@309: {
Chris@309:     double columnDuration = 1.0 / m_colsPerSec;
Chris@309:     int postFilterLatency = int(m_postFilter[0]->getSize() / 2);
Chris@309: 
Chris@309:     return m_startTime + RealTime::fromSeconds
Chris@309:         (columnDuration * (column - postFilterLatency) + 0.02);
Chris@309: }
Chris@309: 
Chris@252: Silvet::Feature
Chris@252: Silvet::makeNoteFeature(int start,
Chris@252:                         int end,
Chris@252:                         int note,
Chris@252:                         int shift,
Chris@320:                         double strength)
Chris@252: {
Chris@252:     Feature f;
Chris@252: 
Chris@252:     f.hasTimestamp = true;
Chris@309:     f.timestamp = getColumnTimestamp(start);
Chris@252: 
Chris@252:     f.hasDuration = true;
Chris@309:     f.duration = getColumnTimestamp(end) - f.timestamp;
Chris@252: 
Chris@252:     f.values.clear();
Chris@336:     f.values.push_back(getNoteFrequency(note, shift));
Chris@320:     f.values.push_back(getVelocityFor(strength, start));
Chris@252: 
Chris@336:     f.label = getNoteName(note, shift);
Chris@252: 
Chris@252:     return f;
Chris@252: }
Chris@252: 
Chris@319: Silvet::Feature
Chris@319: Silvet::makeOnsetFeature(int start,
Chris@319:                          int note,
Chris@319:                          int shift,
Chris@320:                          double strength)
Chris@319: {
Chris@319:     Feature f;
Chris@319: 
Chris@319:     f.hasTimestamp = true;
Chris@319:     f.timestamp = getColumnTimestamp(start);
Chris@319: 
Chris@319:     f.hasDuration = false;
Chris@319: 
Chris@319:     f.values.clear();
Chris@336:     f.values.push_back(getNoteFrequency(note, shift));
Chris@320:     f.values.push_back(getVelocityFor(strength, start));
Chris@319: 
Chris@336:     f.label = getNoteName(note, shift);
Chris@336: 
Chris@336:     return f;
Chris@336: }
Chris@336: 
Chris@336: Silvet::Feature
Chris@336: Silvet::makeOffsetFeature(int col,
Chris@336:                           int note,
Chris@336:                           int shift)
Chris@336: {
Chris@336:     Feature f;
Chris@336:     
Chris@336:     f.hasTimestamp = true;
Chris@336:     f.timestamp = getColumnTimestamp(col);
Chris@336: 
Chris@336:     f.hasDuration = false;
Chris@336: 
Chris@336:     f.values.clear();
Chris@336:     f.values.push_back(getNoteFrequency(note, shift));
Chris@336:     f.values.push_back(0); // velocity 0 for offset
Chris@336: 
Chris@336:     f.label = getNoteName(note, shift) + " off";
Chris@319: 
Chris@319:     return f;
Chris@319: }
Chris@319: 
Chris@320: int
Chris@320: Silvet::getVelocityFor(double strength, int column)
Chris@320: {
Chris@320:     RealTime rt = getColumnTimestamp(column + 1);
Chris@320: 
Chris@320:     float inputGain = getInputGainAt(rt);
Chris@320: 
Chris@320:     double scale = 2.0;
Chris@320:     if (m_mode == LiveMode) scale = 20.0;
Chris@320:         
Chris@320:     double velocity = round((strength * scale) / inputGain);
Chris@320:     
Chris@320:     if (velocity > 127.0) velocity = 127.0;
Chris@320:     if (velocity < 1.0) velocity = 1.0; // assume surpassed 0 threshold already
Chris@320: 
Chris@320:     return int(velocity);
Chris@320: }
Chris@320: 
Chris@252: float
Chris@252: Silvet::getInputGainAt(RealTime t)
Chris@252: {
Chris@252:     map<RealTime, float>::const_iterator i = m_inputGains.lower_bound(t);
Chris@252: 
Chris@252:     if (i == m_inputGains.end()) {
Chris@252:         if (i != m_inputGains.begin()) {
Chris@252:             --i;
Chris@252:         } else {
Chris@252:             return 1.f; // no data
Chris@252:         }
Chris@252:     }
Chris@252: 
Chris@252: //    cerr << "gain at time " << t << " = " << i->second << endl;
Chris@252: 
Chris@252:     return i->second;
Chris@252: }
Chris@252: