Chris@320: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@320: 
Chris@320: /*
Chris@320:     Sonic Visualiser
Chris@320:     An audio file viewer and annotation editor.
Chris@320:     Centre for Digital Music, Queen Mary, University of London.
Chris@320:     This file copyright 2006 Chris Cannam and QMUL.
Chris@320:     
Chris@320:     This program is free software; you can redistribute it and/or
Chris@320:     modify it under the terms of the GNU General Public License as
Chris@320:     published by the Free Software Foundation; either version 2 of the
Chris@320:     License, or (at your option) any later version.  See the file
Chris@320:     COPYING included with this distribution for more information.
Chris@320: */
Chris@320: 
Chris@331: #include "FeatureExtractionModelTransformer.h"
Chris@320: 
Chris@320: #include "plugin/FeatureExtractionPluginFactory.h"
Chris@1225: 
Chris@320: #include "plugin/PluginXml.h"
Chris@475: #include <vamp-hostsdk/Plugin.h>
Chris@320: 
Chris@320: #include "data/model/Model.h"
Chris@320: #include "base/Window.h"
Chris@387: #include "base/Exceptions.h"
Chris@320: #include "data/model/SparseOneDimensionalModel.h"
Chris@320: #include "data/model/SparseTimeValueModel.h"
Chris@1777: #include "data/model/BasicCompressedDenseThreeDimensionalModel.h"
Chris@320: #include "data/model/DenseTimeValueModel.h"
Chris@320: #include "data/model/NoteModel.h"
Chris@441: #include "data/model/RegionModel.h"
Chris@320: #include "data/model/FFTModel.h"
Chris@320: #include "data/model/WaveFileModel.h"
Chris@558: #include "rdf/PluginRDFDescription.h"
Chris@320: 
Chris@350: #include "TransformFactory.h"
Chris@350: 
Chris@320: #include <iostream>
Chris@320: 
Chris@859: #include <QSettings>
Chris@859: 
Chris@1558: //#define DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 1
Chris@1558: 
Chris@350: FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859:                                                                      const Transform &transform) :
Chris@350:     ModelTransformer(in, transform),
Chris@1582:     m_plugin(nullptr),
Chris@1211:     m_haveOutputs(false)
Chris@320: {
Chris@1080:     SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@849: }
Chris@849: 
Chris@849: FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859:                                                                      const Transforms &transforms) :
Chris@849:     ModelTransformer(in, transforms),
Chris@1582:     m_plugin(nullptr),
Chris@1211:     m_haveOutputs(false)
Chris@849: {
Chris@1080:     if (m_transforms.empty()) {
Chris@1080:         SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s)" << endl;
Chris@1080:     } else {
Chris@1080:         SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s), first has plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@1080:     }
Chris@849: }
Chris@849: 
Chris@849: static bool
Chris@849: areTransformsSimilar(const Transform &t1, const Transform &t2)
Chris@849: {
Chris@849:     Transform t2o(t2);
Chris@849:     t2o.setOutput(t1.getOutput());
Chris@849:     return t1 == t2o;
Chris@849: }
Chris@849: 
Chris@849: bool
Chris@849: FeatureExtractionModelTransformer::initialise()
Chris@849: {
Chris@1237:     // This is (now) called from the run thread. The plugin is
Chris@1237:     // constructed, initialised, used, and destroyed all from a single
Chris@1237:     // thread.
Chris@1237:     
Chris@849:     // All transforms must use the same plugin, parameters, and
Chris@849:     // inputs: they can differ only in choice of plugin output. So we
Chris@849:     // initialise based purely on the first transform in the list (but
Chris@849:     // first check that they are actually similar as promised)
Chris@849: 
Chris@1739:     for (int j = 1; in_range_for(m_transforms, j); ++j) {
Chris@849:         if (!areTransformsSimilar(m_transforms[0], m_transforms[j])) {
Chris@849:             m_message = tr("Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output");
Chris@1368:             SVCERR << m_message << endl;
Chris@849:             return false;
Chris@849:         }
Chris@849:     }
Chris@849: 
Chris@849:     Transform primaryTransform = m_transforms[0];
Chris@849: 
Chris@849:     QString pluginId = primaryTransform.getPluginIdentifier();
Chris@320: 
Chris@1226:     FeatureExtractionPluginFactory *factory =
Chris@1226:         FeatureExtractionPluginFactory::instance();
Chris@320: 
Chris@320:     if (!factory) {
Chris@361:         m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@1368:         SVCERR << m_message << endl;
Chris@1429:         return false;
Chris@320:     }
Chris@320: 
Chris@1739:     auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@350:     if (!input) {
Chris@361:         m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@1368:         SVCERR << m_message << endl;
Chris@849:         return false;
Chris@350:     }
Chris@320: 
Chris@1264:     SVDEBUG << "FeatureExtractionModelTransformer: Instantiating plugin for transform in thread "
Chris@1264:             << QThread::currentThreadId() << endl;
Chris@1211:     
Chris@1040:     m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320:     if (!m_plugin) {
Chris@361:         m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@1368:         SVCERR << m_message << endl;
Chris@1429:         return false;
Chris@320:     }
Chris@320: 
Chris@350:     TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@849:         (primaryTransform, m_plugin);
Chris@1368:     
Chris@350:     TransformFactory::getInstance()->setPluginParameters
Chris@849:         (primaryTransform, m_plugin);
Chris@1368:     
Chris@930:     int channelCount = input->getChannelCount();
Chris@930:     if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429:         channelCount = 1;
Chris@320:     }
Chris@930:     if ((int)m_plugin->getMinChannelCount() > channelCount) {
Chris@361:         m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361:             .arg(pluginId)
Chris@361:             .arg(m_plugin->getMinChannelCount())
Chris@361:             .arg(m_plugin->getMaxChannelCount())
Chris@361:             .arg(input->getChannelCount());
Chris@1368:         SVCERR << m_message << endl;
Chris@1429:         return false;
Chris@320:     }
Chris@1374: 
Chris@1374:     int step = primaryTransform.getStepSize();
Chris@1374:     int block = primaryTransform.getBlockSize();
Chris@1368:     
Chris@690:     SVDEBUG << "Initialising feature extraction plugin with channels = "
Chris@1374:             << channelCount << ", step = " << step
Chris@1374:             << ", block = " << block << endl;
Chris@320: 
Chris@1374:     if (!m_plugin->initialise(channelCount, step, block)) {
Chris@1374: 
Chris@1374:         int preferredStep = int(m_plugin->getPreferredStepSize());
Chris@1374:         int preferredBlock = int(m_plugin->getPreferredBlockSize());
Chris@1264:         
Chris@1374:         if (step != preferredStep || block != preferredBlock) {
Chris@361: 
Chris@1374:             SVDEBUG << "Initialisation failed, trying again with preferred step = "
Chris@1374:                     << preferredStep << ", block = " << preferredBlock << endl;
Chris@361:             
Chris@1739:             if (!m_plugin->initialise(channelCount,
Chris@1739:                                       preferredStep,
Chris@1739:                                       preferredBlock)) {
Chris@361: 
Chris@1264:                 SVDEBUG << "Initialisation failed again" << endl;
Chris@1264:                 
Chris@361:                 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368:                 SVCERR << m_message << endl;
Chris@849:                 return false;
Chris@361: 
Chris@361:             } else {
Chris@1264:                 
Chris@1264:                 SVDEBUG << "Initialisation succeeded this time" << endl;
Chris@1374: 
Chris@1374:                 // Set these values into the primary transform in the list
Chris@1374:                 m_transforms[0].setStepSize(preferredStep);
Chris@1374:                 m_transforms[0].setBlockSize(preferredBlock);
Chris@1264:                 
Chris@361:                 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361:                     .arg(pluginId)
Chris@1374:                     .arg(step)
Chris@1374:                     .arg(block)
Chris@1374:                     .arg(preferredStep)
Chris@1374:                     .arg(preferredBlock);
Chris@1368:                 SVCERR << m_message << endl;
Chris@361:             }
Chris@361: 
Chris@361:         } else {
Chris@361: 
Chris@1374:             SVDEBUG << "Initialisation failed (with step = " << step
Chris@1374:                     << " and block = " << block
Chris@1374:                     << ", both matching the plugin's preference)" << endl;
Chris@1264:                 
Chris@361:             m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368:             SVCERR << m_message << endl;
Chris@849:             return false;
Chris@361:         }
Chris@1264:     } else {
Chris@1264:         SVDEBUG << "Initialisation succeeded" << endl;
Chris@320:     }
Chris@320: 
Chris@849:     if (primaryTransform.getPluginVersion() != "") {
Chris@366:         QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@849:         if (pv != primaryTransform.getPluginVersion()) {
Chris@366:             QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@849:                 .arg(primaryTransform.getPluginVersion())
Chris@366:                 .arg(pluginId)
Chris@366:                 .arg(pv);
Chris@366:             if (m_message != "") {
Chris@366:                 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366:             } else {
Chris@366:                 m_message = vm;
Chris@366:             }
Chris@1368:             SVCERR << m_message << endl;
Chris@366:         }
Chris@366:     }
Chris@366: 
Chris@320:     Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320: 
Chris@320:     if (outputs.empty()) {
Chris@361:         m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@1368:         SVCERR << m_message << endl;
Chris@1429:         return false;
Chris@320:     }
Chris@320: 
Chris@1739:     for (int j = 0; in_range_for(m_transforms, j); ++j) {
Chris@849: 
Chris@1739:         for (int i = 0; in_range_for(outputs, i); ++i) {
Chris@1739: 
Chris@849:             if (m_transforms[j].getOutput() == "" ||
Chris@1739:                 outputs[i].identifier ==
Chris@1739:                 m_transforms[j].getOutput().toStdString()) {
Chris@1739:                 
Chris@849:                 m_outputNos.push_back(i);
Chris@1739:                 m_descriptors.push_back(outputs[i]);
Chris@849:                 m_fixedRateFeatureNos.push_back(-1); // we increment before use
Chris@849:                 break;
Chris@849:             }
Chris@849:         }
Chris@849: 
Chris@1739:         if (!in_range_for(m_descriptors, j)) {
Chris@849:             m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@849:                 .arg(pluginId)
Chris@849:                 .arg(m_transforms[j].getOutput());
Chris@1368:             SVCERR << m_message << endl;
Chris@849:             return false;
Chris@849:         }
Chris@320:     }
Chris@320: 
Chris@1739:     for (int j = 0; in_range_for(m_transforms, j); ++j) {
Chris@876:         createOutputModels(j);
Chris@849:     }
Chris@849: 
Chris@1211:     m_outputMutex.lock();
Chris@1211:     m_haveOutputs = true;
Chris@1211:     m_outputsCondition.wakeAll();
Chris@1211:     m_outputMutex.unlock();
Chris@1211: 
Chris@849:     return true;
Chris@558: }
Chris@558: 
Chris@558: void
Chris@1237: FeatureExtractionModelTransformer::deinitialise()
Chris@1237: {
Chris@1264:     SVDEBUG << "FeatureExtractionModelTransformer: deleting plugin for transform in thread "
Chris@1264:             << QThread::currentThreadId() << endl;
Chris@1372: 
Chris@1372:     try {
Chris@1830:         m_plugin = {}; // does not necessarily delete, as it's a
Chris@1830:                        // shared_ptr, but in the design case it will
Chris@1372:     } catch (const std::exception &e) {
Chris@1372:         // A destructor shouldn't throw an exception. But at one point
Chris@1372:         // (now fixed) our plugin stub destructor could have
Chris@1372:         // accidentally done so, so just in case:
Chris@1372:         SVCERR << "FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl;
Chris@1372:         m_message = e.what();
Chris@1372:     }
Chris@1739: 
Chris@1739:     m_descriptors.clear();
Chris@1237: }
Chris@1237: 
Chris@1237: void
Chris@876: FeatureExtractionModelTransformer::createOutputModels(int n)
Chris@558: {
Chris@1739:     auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@1739:     if (!input) return;
Chris@712:     
Chris@849:     PluginRDFDescription description(m_transforms[n].getPluginIdentifier());
Chris@849:     QString outputId = m_transforms[n].getOutput();
Chris@558: 
Chris@320:     int binCount = 1;
Chris@320:     float minValue = 0.0, maxValue = 0.0;
Chris@320:     bool haveExtents = false;
Chris@1739:     bool haveBinCount = m_descriptors[n].hasFixedBinCount;
Chris@876: 
Chris@876:     if (haveBinCount) {
Chris@1739:         binCount = (int)m_descriptors[n].binCount;
Chris@320:     }
Chris@320: 
Chris@876:     m_needAdditionalModels[n] = false;
Chris@876: 
Chris@1739:     if (binCount > 0 && m_descriptors[n].hasKnownExtents) {
Chris@1739:         minValue = m_descriptors[n].minValue;
Chris@1739:         maxValue = m_descriptors[n].maxValue;
Chris@320:         haveExtents = true;
Chris@320:     }
Chris@320: 
Chris@1040:     sv_samplerate_t modelRate = input->getSampleRate();
Chris@1254:     sv_samplerate_t outputRate = modelRate;
Chris@930:     int modelResolution = 1;
Chris@712: 
Chris@1739:     if (m_descriptors[n].sampleType != 
Chris@785:         Vamp::Plugin::OutputDescriptor::OneSamplePerStep) {
Chris@1254: 
Chris@1739:         outputRate = m_descriptors[n].sampleRate;
Chris@1254: 
Chris@1254:         //!!! SV doesn't actually support display of models that have
Chris@1254:         //!!! different underlying rates together -- so we always set
Chris@1254:         //!!! the model rate to be the input model's rate, and adjust
Chris@1254:         //!!! the resolution appropriately.  We can't properly display
Chris@1254:         //!!! data with a higher resolution than the base model at all
Chris@1254:         if (outputRate > input->getSampleRate()) {
Chris@1264:             SVDEBUG << "WARNING: plugin reports output sample rate as "
Chris@1264:                     << outputRate
Chris@1264:                     << " (can't display features with finer resolution than the input rate of "
Chris@1264:                     << modelRate << ")" << endl;
Chris@1254:             outputRate = modelRate;
Chris@785:         }
Chris@785:     }
Chris@785: 
Chris@1739:     switch (m_descriptors[n].sampleType) {
Chris@320: 
Chris@320:     case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@1429:         if (outputRate != 0.0) {
Chris@1429:             modelResolution = int(round(modelRate / outputRate));
Chris@1429:         }
Chris@1429:         break;
Chris@320: 
Chris@320:     case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@1429:         modelResolution = m_transforms[n].getStepSize();
Chris@1429:         break;
Chris@320: 
Chris@320:     case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@1254:         if (outputRate <= 0.0) {
Chris@1739:             SVDEBUG << "WARNING: Fixed sample-rate plugin reports invalid sample rate " << m_descriptors[n].sampleRate << "; defaulting to input rate of " << input->getSampleRate() << endl;
Chris@1071:             modelResolution = 1;
Chris@451:         } else {
Chris@1254:             modelResolution = int(round(modelRate / outputRate));
Chris@1254: //            cerr << "modelRate = " << modelRate << ", descriptor rate = " << outputRate << ", modelResolution = " << modelResolution << endl;
Chris@451:         }
Chris@1429:         break;
Chris@320:     }
Chris@320: 
Chris@441:     bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441: 
Chris@1739:     std::shared_ptr<Model> out;
Chris@849: 
Chris@441:     if (binCount == 0 &&
Chris@1739:         (preDurationPlugin || !m_descriptors[n].hasDuration)) {
Chris@320: 
Chris@445:         // Anything with no value and no duration is an instant
Chris@445: 
Chris@1739:         out = std::make_shared<SparseOneDimensionalModel>
Chris@1739:             (modelRate, modelResolution, false);
Chris@1739: 
Chris@558:         QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849:         out->setRDFTypeURI(outputEventTypeURI);
Chris@558: 
Chris@441:     } else if ((preDurationPlugin && binCount > 1 &&
Chris@1739:                 (m_descriptors[n].sampleType ==
Chris@441:                  Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@1739:                (!preDurationPlugin && m_descriptors[n].hasDuration)) {
Chris@441: 
Chris@441:         // For plugins using the old v1 API without explicit duration,
Chris@441:         // we treat anything that has multiple bins (i.e. that has the
Chris@441:         // potential to have value and duration) and a variable sample
Chris@441:         // rate as a note model, taking its values as pitch, duration
Chris@441:         // and velocity (if present) respectively.  This is the same
Chris@441:         // behaviour as always applied by SV to these plugins in the
Chris@441:         // past.
Chris@441: 
Chris@441:         // For plugins with the newer API, we treat anything with
Chris@441:         // duration as either a note model with pitch and velocity, or
Chris@441:         // a region model.
Chris@441: 
Chris@441:         // How do we know whether it's an interval or note model?
Chris@441:         // What's the essential difference?  Is a note model any
Chris@441:         // interval model using a Hz or "MIDI pitch" scale?  There
Chris@441:         // isn't really a reliable test for "MIDI pitch"...  Does a
Chris@441:         // note model always have velocity?  This is a good question
Chris@441:         // to be addressed by accompanying RDF, but for the moment we
Chris@441:         // will do the following...
Chris@441: 
Chris@441:         bool isNoteModel = false;
Chris@441:         
Chris@441:         // Regions have only value (and duration -- we can't extract a
Chris@441:         // region model from an old-style plugin that doesn't support
Chris@441:         // duration)
Chris@441:         if (binCount > 1) isNoteModel = true;
Chris@441: 
Chris@595:         // Regions do not have units of Hz or MIDI things (a sweeping
Chris@595:         // assumption!)
Chris@1739:         if (m_descriptors[n].unit == "Hz" ||
Chris@1739:             m_descriptors[n].unit.find("MIDI") != std::string::npos ||
Chris@1739:             m_descriptors[n].unit.find("midi") != std::string::npos) {
Chris@595:             isNoteModel = true;
Chris@595:         }
Chris@441: 
Chris@441:         // If we had a "sparse 3D model", we would have the additional
Chris@441:         // problem of determining whether to use that here (if bin
Chris@441:         // count > 1).  But we don't.
Chris@441: 
Chris@859:         QSettings settings;
Chris@859: 
Chris@1647:         if (isNoteModel) {
Chris@441: 
Chris@1769:             QSettings settings;
Chris@1769:             settings.beginGroup("Transformer");
Chris@1769:             bool flexi = settings.value("use-flexi-note-model", false).toBool();
Chris@1769:             settings.endGroup();
Chris@1769: 
Chris@1769:             SVCERR << "flexi = " << flexi << endl;
Chris@1769:             
Chris@441:             NoteModel *model;
Chris@441:             if (haveExtents) {
Chris@859:                 model = new NoteModel
Chris@1769:                     (modelRate, modelResolution, minValue, maxValue, false,
Chris@1769:                      flexi ? NoteModel::FLEXI_NOTE : NoteModel::NORMAL_NOTE);
Chris@441:             } else {
Chris@859:                 model = new NoteModel
Chris@1769:                     (modelRate, modelResolution, false,
Chris@1769:                      flexi ? NoteModel::FLEXI_NOTE : NoteModel::NORMAL_NOTE);
gyorgyf@786:             }
Chris@1739:             model->setScaleUnits(m_descriptors[n].unit.c_str());
Chris@1739:             out.reset(model);
gyorgyf@786: 
Chris@441:         } else {
Chris@441: 
Chris@441:             RegionModel *model;
Chris@441:             if (haveExtents) {
Chris@441:                 model = new RegionModel
Chris@441:                     (modelRate, modelResolution, minValue, maxValue, false);
Chris@441:             } else {
Chris@441:                 model = new RegionModel
Chris@441:                     (modelRate, modelResolution, false);
Chris@441:             }
Chris@1739:             model->setScaleUnits(m_descriptors[n].unit.c_str());
Chris@1739:             out.reset(model);
Chris@441:         }
Chris@441: 
Chris@558:         QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849:         out->setRDFTypeURI(outputEventTypeURI);
Chris@558: 
Chris@876:     } else if (binCount == 1 ||
Chris@1739:                (m_descriptors[n].sampleType == 
Chris@441:                 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441: 
Chris@441:         // Anything that is not a 1D, note, or interval model and that
Chris@441:         // has only one value per result must be a sparse time value
Chris@441:         // model.
Chris@441: 
Chris@441:         // Anything that is not a 1D, note, or interval model and that
Chris@876:         // has a variable sample rate is treated as a set of sparse
Chris@876:         // time value models, one per output bin, because we lack a
Chris@441:         // sparse 3D model.
Chris@320: 
Chris@876:         // Anything that is not a 1D, note, or interval model and that
Chris@876:         // has a fixed sample rate but an unknown number of values per
Chris@876:         // result is also treated as a set of sparse time value models.
Chris@876: 
Chris@876:         // For sets of sparse time value models, we create a single
Chris@876:         // model first as the "standard" output and then create models
Chris@876:         // for bins 1+ in the additional model map (mapping the output
Chris@876:         // descriptor to a list of models indexed by bin-1). But we
Chris@876:         // don't create the additional models yet, as this case has to
Chris@876:         // work even if the number of bins is unknown at this point --
Chris@877:         // we create an additional model (copying its parameters from
Chris@877:         // the default one) each time a new bin is encountered.
Chris@876: 
Chris@876:         if (!haveBinCount || binCount > 1) {
Chris@876:             m_needAdditionalModels[n] = true;
Chris@876:         }
Chris@876: 
Chris@320:         SparseTimeValueModel *model;
Chris@320:         if (haveExtents) {
Chris@320:             model = new SparseTimeValueModel
Chris@320:                 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320:         } else {
Chris@320:             model = new SparseTimeValueModel
Chris@320:                 (modelRate, modelResolution, false);
Chris@320:         }
Chris@558: 
Chris@558:         Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@849:         model->setScaleUnits(outputs[m_outputNos[n]].unit.c_str());
Chris@320: 
Chris@1739:         out.reset(model);
Chris@320: 
Chris@558:         QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849:         out->setRDFTypeURI(outputEventTypeURI);
Chris@558: 
Chris@441:     } else {
Chris@320: 
Chris@441:         // Anything that is not a 1D, note, or interval model and that
Chris@441:         // has a fixed sample rate and more than one value per result
Chris@441:         // must be a dense 3D model.
Chris@320: 
Chris@1777:         auto model =
Chris@1777:             new BasicCompressedDenseThreeDimensionalModel
Chris@1777:             (modelRate, modelResolution, binCount, false);
Chris@320: 
Chris@1739:         if (!m_descriptors[n].binNames.empty()) {
Chris@1429:             std::vector<QString> names;
Chris@1739:             for (int i = 0; i < (int)m_descriptors[n].binNames.size(); ++i) {
Chris@1739:                 names.push_back(m_descriptors[n].binNames[i].c_str());
Chris@1429:             }
Chris@1429:             model->setBinNames(names);
Chris@1429:         }
Chris@320:         
Chris@1739:         out.reset(model);
Chris@558: 
Chris@558:         QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
Chris@849:         out->setRDFTypeURI(outputSignalTypeURI);
Chris@320:     }
Chris@333: 
Chris@849:     if (out) {
Chris@1739:         out->setSourceModel(getInputModel());
Chris@1752:         m_outputs.push_back(ModelById::add(out));
Chris@849:     }
Chris@320: }
Chris@320: 
Chris@1211: void
Chris@1211: FeatureExtractionModelTransformer::awaitOutputModels()
Chris@1211: {
Chris@1211:     m_outputMutex.lock();
Chris@1368:     while (!m_haveOutputs && !m_abandoned) {
Chris@1368:         m_outputsCondition.wait(&m_outputMutex, 500);
Chris@1211:     }
Chris@1211:     m_outputMutex.unlock();
Chris@1211: }
Chris@1211: 
Chris@331: FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320: {
Chris@1237:     // Parent class dtor set the abandoned flag and waited for the run
Chris@1237:     // thread to exit; the run thread owns the plugin, and should have
Chris@1237:     // destroyed it before exiting (via a call to deinitialise)
Chris@320: }
Chris@320: 
Chris@876: FeatureExtractionModelTransformer::Models
Chris@876: FeatureExtractionModelTransformer::getAdditionalOutputModels()
Chris@876: {
Chris@876:     Models mm;
Chris@1739:     for (auto mp : m_additionalModels) {
Chris@1739:         for (auto m: mp.second) {
Chris@1739:             mm.push_back(m.second);
Chris@876:         }
Chris@876:     }
Chris@876:     return mm;
Chris@876: }
Chris@876: 
Chris@877: bool
Chris@877: FeatureExtractionModelTransformer::willHaveAdditionalOutputModels()
Chris@877: {
Chris@1739:     for (auto p : m_needAdditionalModels) {
Chris@1739:         if (p.second) return true;
Chris@877:     }
Chris@877:     return false;
Chris@877: }
Chris@877: 
Chris@1739: ModelId
Chris@876: FeatureExtractionModelTransformer::getAdditionalModel(int n, int binNo)
Chris@876: {
Chris@876:     if (binNo == 0) {
Chris@1739:         SVCERR << "Internal error: binNo == 0 in getAdditionalModel (should be using primary model, not calling getAdditionalModel)" << endl;
Chris@1739:         return {};
Chris@876:     }
Chris@876: 
Chris@1739:     if (!in_range_for(m_outputs, n)) {
Chris@1739:         SVCERR << "getAdditionalModel: Output " << n << " out of range" << endl;
Chris@1739:         return {};
Chris@1739:     }
Chris@876: 
Chris@1739:     if (!in_range_for(m_needAdditionalModels, n) ||
Chris@1739:         !m_needAdditionalModels[n]) {
Chris@1739:         return {};
Chris@1739:     }
Chris@1739:     
Chris@1739:     if (!m_additionalModels[n][binNo].isNone()) {
Chris@1739:         return m_additionalModels[n][binNo];
Chris@1739:     }
Chris@876: 
Chris@1739:     SVDEBUG << "getAdditionalModel(" << n << ", " << binNo
Chris@1739:             << "): creating" << endl;
Chris@876: 
Chris@1739:     auto baseModel = ModelById::getAs<SparseTimeValueModel>(m_outputs[n]);
Chris@1739:     if (!baseModel) {
Chris@1739:         SVCERR << "getAdditionalModel: Output model not conformable, or has vanished" << endl;
Chris@1739:         return {};
Chris@1739:     }
Chris@1739:     
Chris@1739:     SVDEBUG << "getAdditionalModel(" << n << ", " << binNo
Chris@1739:             << "): (from " << baseModel << ")" << endl;
Chris@876: 
Chris@876:     SparseTimeValueModel *additional =
Chris@876:         new SparseTimeValueModel(baseModel->getSampleRate(),
Chris@876:                                  baseModel->getResolution(),
Chris@876:                                  baseModel->getValueMinimum(),
Chris@876:                                  baseModel->getValueMaximum(),
Chris@876:                                  false);
Chris@876: 
Chris@876:     additional->setScaleUnits(baseModel->getScaleUnits());
Chris@876:     additional->setRDFTypeURI(baseModel->getRDFTypeURI());
Chris@876: 
Chris@1752:     ModelId additionalId = ModelById::add
Chris@1752:         (std::shared_ptr<SparseTimeValueModel>(additional));
Chris@1739:     m_additionalModels[n][binNo] = additionalId;
Chris@1739:     return additionalId;
Chris@320: }
Chris@320: 
Chris@320: void
Chris@331: FeatureExtractionModelTransformer::run()
Chris@320: {
Chris@1373:     try {
Chris@1373:         if (!initialise()) {
Chris@1373:             abandon();
Chris@1373:             return;
Chris@1373:         }
Chris@1373:     } catch (const std::exception &e) {
Chris@1368:         abandon();
Chris@1373:         m_message = e.what();
Chris@1368:         return;
Chris@1368:     }
Chris@320: 
Chris@1368:     if (m_outputs.empty()) {
Chris@1368:         abandon();
Chris@1368:         return;
Chris@1368:     }
Chris@320: 
Chris@850:     Transform primaryTransform = m_transforms[0];
Chris@850: 
Chris@1775:     ModelId inputId = getInputModel();
Chris@1775: 
Chris@1739:     bool ready = false;
Chris@1739:     while (!ready && !m_abandoned) {
Chris@1739:         { // scope so as to release input shared_ptr before sleeping
Chris@1775:             auto input = ModelById::getAs<DenseTimeValueModel>(inputId);
Chris@1784:             if (!input || !input->isOK()) {
Chris@1739:                 abandon();
Chris@1739:                 return;
Chris@1739:             }
Chris@1739:             ready = input->isReady();
Chris@1739:         }
Chris@1739:         if (!ready) {
Chris@1775:             SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model "
Chris@1775:                     << inputId << " to be ready..." << endl;
Chris@1739:             usleep(500000);
Chris@1739:         }
Chris@320:     }
Chris@497:     if (m_abandoned) return;
Chris@320: 
Chris@1775: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1775:     SVDEBUG << "FeatureExtractionModelTransformer::run: Input model "
Chris@1775:             << inputId << " is ready, going ahead" << endl;
Chris@1775: #endif
Chris@1739: 
Chris@1755:     sv_samplerate_t sampleRate;
Chris@1755:     int channelCount;
Chris@1755:     sv_frame_t startFrame;
Chris@1755:     sv_frame_t endFrame;
Chris@1755:     
Chris@1755:     { // scope so as not to have this borrowed pointer retained around
Chris@1755:       // the edges of the process loop
Chris@1755:         auto input = ModelById::getAs<DenseTimeValueModel>(inputId);
Chris@1755:         if (!input) {
Chris@1755:             abandon();
Chris@1755:             return;
Chris@1755:         }
Chris@320: 
Chris@1755:         sampleRate = input->getSampleRate();
Chris@1755: 
Chris@1755:         channelCount = input->getChannelCount();
Chris@1755:         if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1755:             channelCount = 1;
Chris@1755:         }
Chris@1755: 
Chris@1755:         startFrame = input->getStartFrame();
Chris@1755:         endFrame = input->getEndFrame();
Chris@320:     }
Chris@320: 
Chris@320:     float **buffers = new float*[channelCount];
Chris@930:     for (int ch = 0; ch < channelCount; ++ch) {
Chris@1429:         buffers[ch] = new float[primaryTransform.getBlockSize() + 2];
Chris@320:     }
Chris@320: 
Chris@930:     int stepSize = primaryTransform.getStepSize();
Chris@930:     int blockSize = primaryTransform.getBlockSize();
Chris@350: 
Chris@320:     bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320:                             Vamp::Plugin::FrequencyDomain);
Chris@1558: 
Chris@320:     std::vector<FFTModel *> fftModels;
Chris@320: 
Chris@320:     if (frequencyDomain) {
Chris@1775: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1775:         SVDEBUG << "FeatureExtractionModelTransformer::run: Input is frequency-domain" << endl;
Chris@1775: #endif
Chris@930:         for (int ch = 0; ch < channelCount; ++ch) {
Chris@320:             FFTModel *model = new FFTModel
Chris@1752:                 (inputId,
Chris@1739:                  channelCount == 1 ? m_input.getChannel() : ch,
Chris@1739:                  primaryTransform.getWindowType(),
Chris@1739:                  blockSize,
Chris@1739:                  stepSize,
Chris@1739:                  blockSize);
Chris@1080:             if (!model->isOK() || model->getError() != "") {
Chris@1080:                 QString err = model->getError();
Chris@320:                 delete model;
Chris@1739:                 for (int j = 0; in_range_for(m_outputNos, j); ++j) {
Chris@850:                     setCompletion(j, 100);
Chris@850:                 }
Chris@1775:                 SVDEBUG << "FeatureExtractionModelTransformer::run: Failed to create FFT model for input model " << inputId << ": " << err << endl;
Chris@1784:                 m_message = "Failed to create the FFT model for this feature extraction model transformer: error is: " + err;
Chris@1784:                 for (int cch = 0; cch < ch; ++cch) {
Chris@1784:                     delete fftModels[cch];
Chris@1784:                 }
Chris@1784:                 abandon();
Chris@1784:                 return;
Chris@320:             }
Chris@320:             fftModels.push_back(model);
Chris@320:         }
Chris@1775: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1775:         SVDEBUG << "FeatureExtractionModelTransformer::run: Created FFT model(s) for frequency-domain input" << endl;
Chris@1775: #endif
Chris@320:     }
Chris@320: 
Chris@850:     RealTime contextStartRT = primaryTransform.getStartTime();
Chris@850:     RealTime contextDurationRT = primaryTransform.getDuration();
Chris@350: 
Chris@1040:     sv_frame_t contextStart =
Chris@350:         RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350: 
Chris@1040:     sv_frame_t contextDuration =
Chris@350:         RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320: 
Chris@320:     if (contextStart == 0 || contextStart < startFrame) {
Chris@320:         contextStart = startFrame;
Chris@320:     }
Chris@320: 
Chris@320:     if (contextDuration == 0) {
Chris@320:         contextDuration = endFrame - contextStart;
Chris@320:     }
Chris@320:     if (contextStart + contextDuration > endFrame) {
Chris@320:         contextDuration = endFrame - contextStart;
Chris@320:     }
Chris@320: 
Chris@1039:     sv_frame_t blockFrame = contextStart;
Chris@320: 
Chris@320:     long prevCompletion = 0;
Chris@320: 
Chris@1739:     for (int j = 0; in_range_for(m_outputNos, j); ++j) {
Chris@850:         setCompletion(j, 0);
Chris@850:     }
Chris@320: 
Chris@1582:     float *reals = nullptr;
Chris@1582:     float *imaginaries = nullptr;
Chris@556:     if (frequencyDomain) {
Chris@556:         reals = new float[blockSize/2 + 1];
Chris@556:         imaginaries = new float[blockSize/2 + 1];
Chris@556:     }
Chris@556: 
Chris@678:     QString error = "";
Chris@678: 
Chris@1372:     try {
Chris@1372:         while (!m_abandoned) {
Chris@320: 
Chris@1372:             if (frequencyDomain) {
Chris@1372:                 if (blockFrame - int(blockSize)/2 >
Chris@1739:                     contextStart + contextDuration) {
Chris@1739:                     break;
Chris@1739:                 }
Chris@1372:             } else {
Chris@1372:                 if (blockFrame >= 
Chris@1739:                     contextStart + contextDuration) {
Chris@1739:                     break;
Chris@1739:                 }
Chris@1372:             }
Chris@320: 
Chris@1558: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558:             SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@1558:                     << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@1558:                     << blockSize << endl;
Chris@1558: #endif
Chris@1558:         
Chris@1372:             int completion = int
Chris@1372:                 ((((blockFrame - contextStart) / stepSize) * 99) /
Chris@1372:                  (contextDuration / stepSize + 1));
Chris@320: 
Chris@1762:             bool haveAllModels = true;
Chris@1755:             if (!ModelById::get(inputId)) {
Chris@1762: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1762:                 SVDEBUG << "FeatureExtractionModelTransformer::run: Input model " << inputId << " no longer exists" << endl;
Chris@1762: #endif
Chris@1762:                 haveAllModels = false;
Chris@1762:             } else {
Chris@1763: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1762:                 SVDEBUG << "Input model " << inputId << " still exists" << endl;
Chris@1763: #endif
Chris@1762:             }
Chris@1762:             for (auto mid: m_outputs) {
Chris@1762:                 if (!ModelById::get(mid)) {
Chris@1762: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1762:                     SVDEBUG << "FeatureExtractionModelTransformer::run: Output model " << mid << " no longer exists" << endl;
Chris@1762: #endif
Chris@1762:                     haveAllModels = false;
Chris@1762:                 } else {
Chris@1762: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1762:                     SVDEBUG << "Output model " << mid << " still exists" << endl;
Chris@1762: #endif
Chris@1762:                 }
Chris@1762:             }
Chris@1762:             if (!haveAllModels) {
Chris@1755:                 abandon();
Chris@1762:                 break;
Chris@1755:             }
Chris@1762:             
Chris@1762: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1762:             SVDEBUG << "FeatureExtractionModelTransformer::run: All models still exist" << endl;
Chris@1762: #endif
Chris@1755: 
Chris@1739:             // channelCount is either input->channelCount or 1
Chris@320: 
Chris@1372:             if (frequencyDomain) {
Chris@1372:                 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1372:                     int column = int((blockFrame - startFrame) / stepSize);
Chris@1372:                     if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) {
Chris@1372:                         for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372:                             buffers[ch][i*2] = reals[i];
Chris@1372:                             buffers[ch][i*2+1] = imaginaries[i];
Chris@1372:                         }
Chris@1372:                     } else {
Chris@1372:                         for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372:                             buffers[ch][i*2] = 0.f;
Chris@1372:                             buffers[ch][i*2+1] = 0.f;
Chris@1372:                         }
Chris@1558:                     }
Chris@1558:                         
Chris@1372:                     error = fftModels[ch]->getError();
Chris@1372:                     if (error != "") {
Chris@1372:                         SVCERR << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << endl;
Chris@1372:                         m_abandoned = true;
Chris@1372:                         m_message = error;
Chris@1372:                         break;
Chris@1008:                     }
Chris@1372:                 }
Chris@1372:             } else {
Chris@1372:                 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@1372:             }
Chris@1372: 
Chris@1372:             if (m_abandoned) break;
Chris@1372: 
Chris@1740:             auto features = m_plugin->process
Chris@1558:                 (buffers,
Chris@1558:                  RealTime::frame2RealTime(blockFrame, sampleRate)
Chris@1558:                  .toVampRealTime());
Chris@1558:             
Chris@1372:             if (m_abandoned) break;
Chris@1372: 
Chris@1740:             for (int j = 0; in_range_for(m_outputNos, j); ++j) {
Chris@1740:                 for (int fi = 0; in_range_for(features[m_outputNos[j]], fi); ++fi) {
Chris@1740:                     auto feature = features[m_outputNos[j]][fi];
Chris@1372:                     addFeature(j, blockFrame, feature);
Chris@678:                 }
Chris@363:             }
Chris@1372: 
Chris@1372:             if (blockFrame == contextStart || completion > prevCompletion) {
Chris@1740:                 for (int j = 0; in_range_for(m_outputNos, j); ++j) {
Chris@1372:                     setCompletion(j, completion);
Chris@1372:                 }
Chris@1372:                 prevCompletion = completion;
Chris@1372:             }
Chris@1372: 
Chris@1372:             blockFrame += stepSize;
Chris@1372: 
Chris@320:         }
Chris@320: 
Chris@1372:         if (!m_abandoned) {
Chris@1740:             auto features = m_plugin->getRemainingFeatures();
Chris@497: 
Chris@1740:             for (int j = 0; in_range_for(m_outputNos, j); ++j) {
Chris@1740:                 for (int fi = 0; in_range_for(features[m_outputNos[j]], fi); ++fi) {
Chris@1740:                     auto feature = features[m_outputNos[j]][fi];
Chris@1372:                     addFeature(j, blockFrame, feature);
Chris@1762:                     if (m_abandoned) {
Chris@1762:                         break;
Chris@1762:                     }
Chris@1372:                 }
Chris@850:             }
Chris@850:         }
Chris@1372:     } catch (const std::exception &e) {
Chris@1372:         SVCERR << "FeatureExtractionModelTransformer::run: Exception caught: "
Chris@1372:                << e.what() << endl;
Chris@1372:         m_abandoned = true;
Chris@1372:         m_message = e.what();
Chris@497:     }
Chris@320: 
Chris@850:     for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850:         setCompletion(j, 100);
Chris@850:     }
Chris@320: 
Chris@320:     if (frequencyDomain) {
Chris@930:         for (int ch = 0; ch < channelCount; ++ch) {
Chris@320:             delete fftModels[ch];
Chris@320:         }
Chris@556:         delete[] reals;
Chris@556:         delete[] imaginaries;
Chris@320:     }
Chris@974: 
Chris@974:     for (int ch = 0; ch < channelCount; ++ch) {
Chris@974:         delete[] buffers[ch];
Chris@974:     }
Chris@974:     delete[] buffers;
Chris@1237: 
Chris@1237:     deinitialise();
Chris@320: }
Chris@320: 
Chris@320: void
Chris@363: FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@1039:                                              sv_frame_t startFrame,
Chris@1039:                                              sv_frame_t size,
Chris@363:                                              float **buffers)
Chris@320: {
Chris@1039:     sv_frame_t offset = 0;
Chris@320: 
Chris@320:     if (startFrame < 0) {
Chris@363:         for (int c = 0; c < channelCount; ++c) {
Chris@1039:             for (sv_frame_t i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363:                 buffers[c][i] = 0.0f;
Chris@363:             }
Chris@320:         }
Chris@320:         offset = -startFrame;
Chris@320:         size -= offset;
Chris@320:         if (size <= 0) return;
Chris@320:         startFrame = 0;
Chris@320:     }
Chris@320: 
Chris@1739:     auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@1739:     if (!input) {
Chris@1739:         return;
Chris@1739:     }
Chris@363:     
Chris@1039:     sv_frame_t got = 0;
Chris@350: 
Chris@363:     if (channelCount == 1) {
Chris@363: 
Chris@1096:         auto data = input->getData(m_input.getChannel(), startFrame, size);
Chris@1096:         got = data.size();
Chris@1096: 
Chris@1096:         copy(data.begin(), data.end(), buffers[0] + offset);
Chris@363: 
Chris@363:         if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363:             // use mean instead of sum, as plugin input
Chris@363:             float cc = float(input->getChannelCount());
Chris@1096:             for (sv_frame_t i = 0; i < got; ++i) {
Chris@363:                 buffers[0][i + offset] /= cc;
Chris@363:             }
Chris@363:         }
Chris@363: 
Chris@363:     } else {
Chris@363: 
Chris@1096:         auto data = input->getMultiChannelData(0, channelCount-1, startFrame, size);
Chris@1096:         if (!data.empty()) {
Chris@1096:             got = data[0].size();
Chris@1096:             for (int c = 0; in_range_for(data, c); ++c) {
Chris@1096:                 copy(data[c].begin(), data[c].end(), buffers[c] + offset);
Chris@363:             }
Chris@363:         }
Chris@363:     }
Chris@320: 
Chris@320:     while (got < size) {
Chris@363:         for (int c = 0; c < channelCount; ++c) {
Chris@363:             buffers[c][got + offset] = 0.0;
Chris@363:         }
Chris@320:         ++got;
Chris@320:     }
Chris@320: }
Chris@320: 
Chris@320: void
Chris@850: FeatureExtractionModelTransformer::addFeature(int n,
Chris@1039:                                               sv_frame_t blockFrame,
Chris@850:                                               const Vamp::Plugin::Feature &feature)
Chris@320: {
Chris@1739:     auto input = ModelById::get(getInputModel());
Chris@1739:     if (!input) return;
Chris@1739: 
Chris@1739:     sv_samplerate_t inputRate = input->getSampleRate();
Chris@320: 
Chris@843: //    cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
Chris@712: //              << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
Chris@712: //              << ", timestamp = " << feature.timestamp << ", hasDuration = "
Chris@712: //              << feature.hasDuration << ", duration = " << feature.duration
Chris@843: //              << endl;
Chris@320: 
Chris@1039:     sv_frame_t frame = blockFrame;
Chris@320: 
Chris@1739:     if (m_descriptors[n].sampleType ==
Chris@1429:         Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320: 
Chris@1429:         if (!feature.hasTimestamp) {
Chris@1429:             SVDEBUG
Chris@1429:                 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@1429:                 << "Feature has variable sample rate but no timestamp!"
Chris@1429:                 << endl;
Chris@1429:             return;
Chris@1429:         } else {
Chris@1429:             frame = RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@1429:         }
Chris@320: 
Chris@1071: //        cerr << "variable sample rate: timestamp = " << feature.timestamp
Chris@1071: //             << " at input rate " << inputRate << " -> " << frame << endl;
Chris@1071:         
Chris@1739:     } else if (m_descriptors[n].sampleType ==
Chris@1429:                Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320: 
Chris@1739:         sv_samplerate_t rate = m_descriptors[n].sampleRate;
Chris@1071:         if (rate <= 0.0) {
Chris@1071:             rate = inputRate;
Chris@1071:         }
Chris@1071:         
Chris@779:         if (!feature.hasTimestamp) {
Chris@849:             ++m_fixedRateFeatureNos[n];
Chris@779:         } else {
Chris@779:             RealTime ts(feature.timestamp.sec, feature.timestamp.nsec);
Chris@1071:             m_fixedRateFeatureNos[n] = (int)lrint(ts.toDouble() * rate);
Chris@779:         }
Chris@862: 
Chris@1071: //        cerr << "m_fixedRateFeatureNo = " << m_fixedRateFeatureNos[n]
Chris@1739: //             << ", m_descriptor->sampleRate = " << m_descriptors[n].sampleRate
Chris@862: //             << ", inputRate = " << inputRate
Chris@862: //             << " giving frame = ";
Chris@1071:         frame = lrint((double(m_fixedRateFeatureNos[n]) / rate) * inputRate);
Chris@1071: //        cerr << frame << endl;
Chris@320:     }
Chris@862: 
Chris@862:     if (frame < 0) {
Chris@1264:         SVDEBUG
Chris@862:             << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@862:             << "Negative frame counts are not supported (frame = " << frame
Chris@862:             << " from timestamp " << feature.timestamp
Chris@862:             << "), dropping feature" 
Chris@862:             << endl;
Chris@862:         return;
Chris@862:     }
Chris@862: 
Chris@441:     // Rather than repeat the complicated tests from the constructor
Chris@441:     // to determine what sort of model we must be adding the features
Chris@441:     // to, we instead test what sort of model the constructor decided
Chris@441:     // to create.
Chris@320: 
Chris@1739:     ModelId outputId = m_outputs[n];
Chris@1740: 
Chris@1740:     if (isOutputType<SparseOneDimensionalModel>(n)) {
Chris@1740: 
Chris@1740:         auto model = ModelById::getAs<SparseOneDimensionalModel>(outputId);
Chris@1740:         if (!model) return;
Chris@1740:         model->add(Event(frame, feature.label.c_str()));
Chris@1740:         
Chris@1740:     } else if (isOutputType<SparseTimeValueModel>(n)) {
Chris@1740: 
Chris@1740:         auto model = ModelById::getAs<SparseTimeValueModel>(outputId);
Chris@1740:         if (!model) return;
Chris@1740: 
Chris@1740:         for (int i = 0; in_range_for(feature.values, i); ++i) {
Chris@1740: 
Chris@1740:             float value = feature.values[i];
Chris@1740: 
Chris@1740:             QString label = feature.label.c_str();
Chris@1740:             if (feature.values.size() > 1) {
Chris@1740:                 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@1740:             }
Chris@1740: 
Chris@1740:             auto targetModel = model;
Chris@1740: 
Chris@1740:             if (m_needAdditionalModels[n] && i > 0) {
Chris@1740:                 targetModel = ModelById::getAs<SparseTimeValueModel>
Chris@1740:                     (getAdditionalModel(n, i));
Chris@1740:                 if (!targetModel) targetModel = model;
Chris@1740:             }
Chris@1740: 
Chris@1740:             targetModel->add(Event(frame, value, label));
Chris@1740:         }
Chris@1740: 
Chris@1740:     } else if (isOutputType<NoteModel>(n) || isOutputType<RegionModel>(n)) {
Chris@1739:     
Chris@1740:         int index = 0;
Chris@1740: 
Chris@1740:         float value = 0.0;
Chris@1740:         if ((int)feature.values.size() > index) {
Chris@1740:             value = feature.values[index++];
Chris@1739:         }
Chris@441: 
Chris@1740:         sv_frame_t duration = 1;
Chris@1740:         if (feature.hasDuration) {
Chris@1740:             duration = RealTime::realTime2Frame(feature.duration, inputRate);
Chris@1740:         } else {
Chris@1740:             if (in_range_for(feature.values, index)) {
Chris@1740:                 duration = lrintf(feature.values[index++]);
Chris@1739:             }
Chris@1739:         }
Chris@1739: 
Chris@1740:         auto noteModel = ModelById::getAs<NoteModel>(outputId);
Chris@1740:         if (noteModel) {
Chris@1739: 
Chris@1740:             float velocity = 100;
Chris@1739:             if ((int)feature.values.size() > index) {
Chris@1740:                 velocity = feature.values[index++];
Chris@454:             }
Chris@1740:             if (velocity < 0) velocity = 127;
Chris@1740:             if (velocity > 127) velocity = 127;
Chris@1740:             
Chris@1740:             noteModel->add(Event(frame, value, // value is pitch
Chris@1740:                                  duration,
Chris@1740:                                  velocity / 127.f,
Chris@1740:                                  feature.label.c_str()));
Chris@1740:         }
Chris@454: 
Chris@1740:         auto regionModel = ModelById::getAs<RegionModel>(outputId);
Chris@1740:         if (regionModel) {
Chris@1740:             
Chris@1740:             if (feature.hasDuration && !feature.values.empty()) {
Chris@1740:                 
Chris@1740:                 for (int i = 0; in_range_for(feature.values, i); ++i) {
Chris@1740:                     
Chris@1740:                     float value = feature.values[i];
Chris@1740:                     
Chris@1740:                     QString label = feature.label.c_str();
Chris@1740:                     if (feature.values.size() > 1) {
Chris@1740:                         label = QString("[%1] %2").arg(i+1).arg(label);
Chris@1739:                     }
Chris@1740:                     
Chris@1739:                     regionModel->add(Event(frame,
Chris@1739:                                            value,
Chris@1739:                                            duration,
Chris@1740:                                            label));
Chris@1739:                 }
Chris@1740:             } else {
Chris@1740:                 
Chris@1740:                 regionModel->add(Event(frame,
Chris@1740:                                        value,
Chris@1740:                                        duration,
Chris@1740:                                        feature.label.c_str()));
Chris@441:             }
Chris@441:         }
Chris@1740: 
Chris@1777:     } else if (isOutputType<BasicCompressedDenseThreeDimensionalModel>(n)) {
Chris@1740: 
Chris@1740:         auto model = ModelById::getAs
Chris@1777:             <BasicCompressedDenseThreeDimensionalModel>(outputId);
Chris@1740:         if (!model) return;
Chris@1740:         
Chris@1837:         DenseThreeDimensionalModel::Column values;
Chris@1837:         values.insert(values.begin(),
Chris@1837:                       feature.values.begin(), feature.values.end());
Chris@1740:         
Chris@1740:         if (!feature.hasTimestamp && m_fixedRateFeatureNos[n] >= 0) {
Chris@1740:             model->setColumn(m_fixedRateFeatureNos[n], values);
Chris@1740:         } else {
Chris@1740:             model->setColumn(int(frame / model->getResolution()), values);
Chris@1740:         }
Chris@1762:     } else {
Chris@1762:         
Chris@1762:         SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type - possibly a deleted model" << endl;
Chris@1762:         abandon();
Chris@1739:     }
Chris@320: }
Chris@320: 
Chris@320: void
Chris@850: FeatureExtractionModelTransformer::setCompletion(int n, int completion)
Chris@320: {
Chris@1558: #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558:     SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
Chris@1558:               << completion << ")" << endl;
Chris@1558: #endif
Chris@320: 
Chris@1740:     (void)
Chris@1740:         (setOutputCompletion<SparseOneDimensionalModel>(n, completion) ||
Chris@1740:          setOutputCompletion<SparseTimeValueModel>(n, completion) ||
Chris@1740:          setOutputCompletion<NoteModel>(n, completion) ||
Chris@1740:          setOutputCompletion<RegionModel>(n, completion) ||
Chris@1777:          setOutputCompletion<BasicCompressedDenseThreeDimensionalModel>(n, completion));
Chris@320: }
Chris@320: