annotate transform/FeatureExtractionModelTransformer.cpp @ 1651:7a56bb85030f single-point

Introduce deferred notifier, + start converting sparse time-value model (perhaps we should rename it too)
author Chris Cannam
date Mon, 18 Mar 2019 14:17:20 +0000
parents 1cc9a0d4b1b6
children 5b7b01da430a
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@1225 19
Chris@320 20 #include "plugin/PluginXml.h"
Chris@475 21 #include <vamp-hostsdk/Plugin.h>
Chris@320 22
Chris@320 23 #include "data/model/Model.h"
Chris@320 24 #include "base/Window.h"
Chris@387 25 #include "base/Exceptions.h"
Chris@320 26 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 27 #include "data/model/SparseTimeValueModel.h"
Chris@320 28 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 29 #include "data/model/DenseTimeValueModel.h"
Chris@320 30 #include "data/model/NoteModel.h"
Chris@441 31 #include "data/model/RegionModel.h"
Chris@320 32 #include "data/model/FFTModel.h"
Chris@320 33 #include "data/model/WaveFileModel.h"
Chris@558 34 #include "rdf/PluginRDFDescription.h"
Chris@320 35
Chris@350 36 #include "TransformFactory.h"
Chris@350 37
Chris@320 38 #include <iostream>
Chris@320 39
Chris@859 40 #include <QSettings>
Chris@859 41
Chris@1558 42 //#define DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 1
Chris@1558 43
Chris@350 44 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 45 const Transform &transform) :
Chris@350 46 ModelTransformer(in, transform),
Chris@1582 47 m_plugin(nullptr),
Chris@1211 48 m_haveOutputs(false)
Chris@320 49 {
Chris@1080 50 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@849 51 }
Chris@849 52
Chris@849 53 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 54 const Transforms &transforms) :
Chris@849 55 ModelTransformer(in, transforms),
Chris@1582 56 m_plugin(nullptr),
Chris@1211 57 m_haveOutputs(false)
Chris@849 58 {
Chris@1080 59 if (m_transforms.empty()) {
Chris@1080 60 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s)" << endl;
Chris@1080 61 } else {
Chris@1080 62 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s), first has plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@1080 63 }
Chris@849 64 }
Chris@849 65
Chris@849 66 static bool
Chris@849 67 areTransformsSimilar(const Transform &t1, const Transform &t2)
Chris@849 68 {
Chris@849 69 Transform t2o(t2);
Chris@849 70 t2o.setOutput(t1.getOutput());
Chris@849 71 return t1 == t2o;
Chris@849 72 }
Chris@849 73
Chris@849 74 bool
Chris@849 75 FeatureExtractionModelTransformer::initialise()
Chris@849 76 {
Chris@1237 77 // This is (now) called from the run thread. The plugin is
Chris@1237 78 // constructed, initialised, used, and destroyed all from a single
Chris@1237 79 // thread.
Chris@1237 80
Chris@849 81 // All transforms must use the same plugin, parameters, and
Chris@849 82 // inputs: they can differ only in choice of plugin output. So we
Chris@849 83 // initialise based purely on the first transform in the list (but
Chris@849 84 // first check that they are actually similar as promised)
Chris@849 85
Chris@849 86 for (int j = 1; j < (int)m_transforms.size(); ++j) {
Chris@849 87 if (!areTransformsSimilar(m_transforms[0], m_transforms[j])) {
Chris@849 88 m_message = tr("Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output");
Chris@1368 89 SVCERR << m_message << endl;
Chris@849 90 return false;
Chris@849 91 }
Chris@849 92 }
Chris@849 93
Chris@849 94 Transform primaryTransform = m_transforms[0];
Chris@849 95
Chris@849 96 QString pluginId = primaryTransform.getPluginIdentifier();
Chris@320 97
Chris@1226 98 FeatureExtractionPluginFactory *factory =
Chris@1226 99 FeatureExtractionPluginFactory::instance();
Chris@320 100
Chris@320 101 if (!factory) {
Chris@361 102 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@1368 103 SVCERR << m_message << endl;
Chris@1429 104 return false;
Chris@320 105 }
Chris@320 106
Chris@350 107 DenseTimeValueModel *input = getConformingInput();
Chris@350 108 if (!input) {
Chris@361 109 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@1368 110 SVCERR << m_message << endl;
Chris@849 111 return false;
Chris@350 112 }
Chris@320 113
Chris@1264 114 SVDEBUG << "FeatureExtractionModelTransformer: Instantiating plugin for transform in thread "
Chris@1264 115 << QThread::currentThreadId() << endl;
Chris@1211 116
Chris@1040 117 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 118 if (!m_plugin) {
Chris@361 119 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@1368 120 SVCERR << m_message << endl;
Chris@1429 121 return false;
Chris@320 122 }
Chris@320 123
Chris@350 124 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@849 125 (primaryTransform, m_plugin);
Chris@1368 126
Chris@350 127 TransformFactory::getInstance()->setPluginParameters
Chris@849 128 (primaryTransform, m_plugin);
Chris@1368 129
Chris@930 130 int channelCount = input->getChannelCount();
Chris@930 131 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 132 channelCount = 1;
Chris@320 133 }
Chris@930 134 if ((int)m_plugin->getMinChannelCount() > channelCount) {
Chris@361 135 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 136 .arg(pluginId)
Chris@361 137 .arg(m_plugin->getMinChannelCount())
Chris@361 138 .arg(m_plugin->getMaxChannelCount())
Chris@361 139 .arg(input->getChannelCount());
Chris@1368 140 SVCERR << m_message << endl;
Chris@1429 141 return false;
Chris@320 142 }
Chris@1374 143
Chris@1374 144 int step = primaryTransform.getStepSize();
Chris@1374 145 int block = primaryTransform.getBlockSize();
Chris@1368 146
Chris@690 147 SVDEBUG << "Initialising feature extraction plugin with channels = "
Chris@1374 148 << channelCount << ", step = " << step
Chris@1374 149 << ", block = " << block << endl;
Chris@320 150
Chris@1374 151 if (!m_plugin->initialise(channelCount, step, block)) {
Chris@1374 152
Chris@1374 153 int preferredStep = int(m_plugin->getPreferredStepSize());
Chris@1374 154 int preferredBlock = int(m_plugin->getPreferredBlockSize());
Chris@1264 155
Chris@1374 156 if (step != preferredStep || block != preferredBlock) {
Chris@361 157
Chris@1374 158 SVDEBUG << "Initialisation failed, trying again with preferred step = "
Chris@1374 159 << preferredStep << ", block = " << preferredBlock << endl;
Chris@361 160
Chris@1374 161 if (!m_plugin->initialise(channelCount, preferredStep, preferredBlock)) {
Chris@361 162
Chris@1264 163 SVDEBUG << "Initialisation failed again" << endl;
Chris@1264 164
Chris@361 165 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 166 SVCERR << m_message << endl;
Chris@849 167 return false;
Chris@361 168
Chris@361 169 } else {
Chris@1264 170
Chris@1264 171 SVDEBUG << "Initialisation succeeded this time" << endl;
Chris@1374 172
Chris@1374 173 // Set these values into the primary transform in the list
Chris@1374 174 m_transforms[0].setStepSize(preferredStep);
Chris@1374 175 m_transforms[0].setBlockSize(preferredBlock);
Chris@1264 176
Chris@361 177 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 178 .arg(pluginId)
Chris@1374 179 .arg(step)
Chris@1374 180 .arg(block)
Chris@1374 181 .arg(preferredStep)
Chris@1374 182 .arg(preferredBlock);
Chris@1368 183 SVCERR << m_message << endl;
Chris@361 184 }
Chris@361 185
Chris@361 186 } else {
Chris@361 187
Chris@1374 188 SVDEBUG << "Initialisation failed (with step = " << step
Chris@1374 189 << " and block = " << block
Chris@1374 190 << ", both matching the plugin's preference)" << endl;
Chris@1264 191
Chris@361 192 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 193 SVCERR << m_message << endl;
Chris@849 194 return false;
Chris@361 195 }
Chris@1264 196 } else {
Chris@1264 197 SVDEBUG << "Initialisation succeeded" << endl;
Chris@320 198 }
Chris@320 199
Chris@849 200 if (primaryTransform.getPluginVersion() != "") {
Chris@366 201 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@849 202 if (pv != primaryTransform.getPluginVersion()) {
Chris@366 203 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@849 204 .arg(primaryTransform.getPluginVersion())
Chris@366 205 .arg(pluginId)
Chris@366 206 .arg(pv);
Chris@366 207 if (m_message != "") {
Chris@366 208 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 209 } else {
Chris@366 210 m_message = vm;
Chris@366 211 }
Chris@1368 212 SVCERR << m_message << endl;
Chris@366 213 }
Chris@366 214 }
Chris@366 215
Chris@320 216 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 217
Chris@320 218 if (outputs.empty()) {
Chris@361 219 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@1368 220 SVCERR << m_message << endl;
Chris@1429 221 return false;
Chris@320 222 }
Chris@320 223
Chris@849 224 for (int j = 0; j < (int)m_transforms.size(); ++j) {
Chris@849 225
Chris@849 226 for (int i = 0; i < (int)outputs.size(); ++i) {
Chris@849 227 // SVDEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl;
Chris@849 228 if (m_transforms[j].getOutput() == "" ||
Chris@849 229 outputs[i].identifier == m_transforms[j].getOutput().toStdString()) {
Chris@849 230 m_outputNos.push_back(i);
Chris@849 231 m_descriptors.push_back(new Vamp::Plugin::OutputDescriptor(outputs[i]));
Chris@849 232 m_fixedRateFeatureNos.push_back(-1); // we increment before use
Chris@849 233 break;
Chris@849 234 }
Chris@849 235 }
Chris@849 236
Chris@930 237 if ((int)m_descriptors.size() <= j) {
Chris@849 238 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@849 239 .arg(pluginId)
Chris@849 240 .arg(m_transforms[j].getOutput());
Chris@1368 241 SVCERR << m_message << endl;
Chris@849 242 return false;
Chris@849 243 }
Chris@320 244 }
Chris@320 245
Chris@849 246 for (int j = 0; j < (int)m_transforms.size(); ++j) {
Chris@876 247 createOutputModels(j);
Chris@849 248 }
Chris@849 249
Chris@1211 250 m_outputMutex.lock();
Chris@1211 251 m_haveOutputs = true;
Chris@1211 252 m_outputsCondition.wakeAll();
Chris@1211 253 m_outputMutex.unlock();
Chris@1211 254
Chris@849 255 return true;
Chris@558 256 }
Chris@558 257
Chris@558 258 void
Chris@1237 259 FeatureExtractionModelTransformer::deinitialise()
Chris@1237 260 {
Chris@1264 261 SVDEBUG << "FeatureExtractionModelTransformer: deleting plugin for transform in thread "
Chris@1264 262 << QThread::currentThreadId() << endl;
Chris@1372 263
Chris@1372 264 try {
Chris@1372 265 delete m_plugin;
Chris@1372 266 } catch (const std::exception &e) {
Chris@1372 267 // A destructor shouldn't throw an exception. But at one point
Chris@1372 268 // (now fixed) our plugin stub destructor could have
Chris@1372 269 // accidentally done so, so just in case:
Chris@1372 270 SVCERR << "FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl;
Chris@1372 271 m_message = e.what();
Chris@1372 272 }
Chris@1582 273 m_plugin = nullptr;
Chris@1372 274
Chris@1237 275 for (int j = 0; j < (int)m_descriptors.size(); ++j) {
Chris@1237 276 delete m_descriptors[j];
Chris@1237 277 }
Chris@1237 278 }
Chris@1237 279
Chris@1237 280 void
Chris@876 281 FeatureExtractionModelTransformer::createOutputModels(int n)
Chris@558 282 {
Chris@558 283 DenseTimeValueModel *input = getConformingInput();
Chris@712 284
Chris@849 285 PluginRDFDescription description(m_transforms[n].getPluginIdentifier());
Chris@849 286 QString outputId = m_transforms[n].getOutput();
Chris@558 287
Chris@320 288 int binCount = 1;
Chris@320 289 float minValue = 0.0, maxValue = 0.0;
Chris@320 290 bool haveExtents = false;
Chris@876 291 bool haveBinCount = m_descriptors[n]->hasFixedBinCount;
Chris@876 292
Chris@876 293 if (haveBinCount) {
Chris@1429 294 binCount = (int)m_descriptors[n]->binCount;
Chris@320 295 }
Chris@320 296
Chris@876 297 m_needAdditionalModels[n] = false;
Chris@876 298
Chris@843 299 // cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@1429 300 // << binCount << endl;
Chris@320 301
Chris@849 302 if (binCount > 0 && m_descriptors[n]->hasKnownExtents) {
Chris@1429 303 minValue = m_descriptors[n]->minValue;
Chris@1429 304 maxValue = m_descriptors[n]->maxValue;
Chris@320 305 haveExtents = true;
Chris@320 306 }
Chris@320 307
Chris@1040 308 sv_samplerate_t modelRate = input->getSampleRate();
Chris@1254 309 sv_samplerate_t outputRate = modelRate;
Chris@930 310 int modelResolution = 1;
Chris@712 311
Chris@849 312 if (m_descriptors[n]->sampleType !=
Chris@785 313 Vamp::Plugin::OutputDescriptor::OneSamplePerStep) {
Chris@1254 314
Chris@1254 315 outputRate = m_descriptors[n]->sampleRate;
Chris@1254 316
Chris@1254 317 //!!! SV doesn't actually support display of models that have
Chris@1254 318 //!!! different underlying rates together -- so we always set
Chris@1254 319 //!!! the model rate to be the input model's rate, and adjust
Chris@1254 320 //!!! the resolution appropriately. We can't properly display
Chris@1254 321 //!!! data with a higher resolution than the base model at all
Chris@1254 322 if (outputRate > input->getSampleRate()) {
Chris@1264 323 SVDEBUG << "WARNING: plugin reports output sample rate as "
Chris@1264 324 << outputRate
Chris@1264 325 << " (can't display features with finer resolution than the input rate of "
Chris@1264 326 << modelRate << ")" << endl;
Chris@1254 327 outputRate = modelRate;
Chris@785 328 }
Chris@785 329 }
Chris@785 330
Chris@849 331 switch (m_descriptors[n]->sampleType) {
Chris@320 332
Chris@320 333 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@1429 334 if (outputRate != 0.0) {
Chris@1429 335 modelResolution = int(round(modelRate / outputRate));
Chris@1429 336 }
Chris@1429 337 break;
Chris@320 338
Chris@320 339 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@1429 340 modelResolution = m_transforms[n].getStepSize();
Chris@1429 341 break;
Chris@320 342
Chris@320 343 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@1254 344 if (outputRate <= 0.0) {
Chris@1264 345 SVDEBUG << "WARNING: Fixed sample-rate plugin reports invalid sample rate " << m_descriptors[n]->sampleRate << "; defaulting to input rate of " << input->getSampleRate() << endl;
Chris@1071 346 modelResolution = 1;
Chris@451 347 } else {
Chris@1254 348 modelResolution = int(round(modelRate / outputRate));
Chris@1254 349 // cerr << "modelRate = " << modelRate << ", descriptor rate = " << outputRate << ", modelResolution = " << modelResolution << endl;
Chris@451 350 }
Chris@1429 351 break;
Chris@320 352 }
Chris@320 353
Chris@441 354 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 355
Chris@1582 356 Model *out = nullptr;
Chris@849 357
Chris@441 358 if (binCount == 0 &&
Chris@849 359 (preDurationPlugin || !m_descriptors[n]->hasDuration)) {
Chris@320 360
Chris@445 361 // Anything with no value and no duration is an instant
Chris@445 362
Chris@849 363 out = new SparseOneDimensionalModel(modelRate, modelResolution, false);
Chris@558 364 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 365 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 366
Chris@441 367 } else if ((preDurationPlugin && binCount > 1 &&
Chris@849 368 (m_descriptors[n]->sampleType ==
Chris@441 369 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@849 370 (!preDurationPlugin && m_descriptors[n]->hasDuration)) {
Chris@441 371
Chris@441 372 // For plugins using the old v1 API without explicit duration,
Chris@441 373 // we treat anything that has multiple bins (i.e. that has the
Chris@441 374 // potential to have value and duration) and a variable sample
Chris@441 375 // rate as a note model, taking its values as pitch, duration
Chris@441 376 // and velocity (if present) respectively. This is the same
Chris@441 377 // behaviour as always applied by SV to these plugins in the
Chris@441 378 // past.
Chris@441 379
Chris@441 380 // For plugins with the newer API, we treat anything with
Chris@441 381 // duration as either a note model with pitch and velocity, or
Chris@441 382 // a region model.
Chris@441 383
Chris@441 384 // How do we know whether it's an interval or note model?
Chris@441 385 // What's the essential difference? Is a note model any
Chris@441 386 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 387 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 388 // note model always have velocity? This is a good question
Chris@441 389 // to be addressed by accompanying RDF, but for the moment we
Chris@441 390 // will do the following...
Chris@441 391
Chris@441 392 bool isNoteModel = false;
Chris@441 393
Chris@441 394 // Regions have only value (and duration -- we can't extract a
Chris@441 395 // region model from an old-style plugin that doesn't support
Chris@441 396 // duration)
Chris@441 397 if (binCount > 1) isNoteModel = true;
Chris@441 398
Chris@595 399 // Regions do not have units of Hz or MIDI things (a sweeping
Chris@595 400 // assumption!)
Chris@849 401 if (m_descriptors[n]->unit == "Hz" ||
Chris@849 402 m_descriptors[n]->unit.find("MIDI") != std::string::npos ||
Chris@849 403 m_descriptors[n]->unit.find("midi") != std::string::npos) {
Chris@595 404 isNoteModel = true;
Chris@595 405 }
Chris@441 406
Chris@441 407 // If we had a "sparse 3D model", we would have the additional
Chris@441 408 // problem of determining whether to use that here (if bin
Chris@441 409 // count > 1). But we don't.
Chris@441 410
Chris@859 411 QSettings settings;
Chris@859 412
Chris@1647 413 if (isNoteModel) {
Chris@441 414
Chris@441 415 NoteModel *model;
Chris@441 416 if (haveExtents) {
Chris@859 417 model = new NoteModel
Chris@859 418 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 419 } else {
Chris@859 420 model = new NoteModel
Chris@859 421 (modelRate, modelResolution, false);
gyorgyf@786 422 }
Chris@849 423 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 424 out = model;
gyorgyf@786 425
Chris@441 426 } else {
Chris@441 427
Chris@441 428 RegionModel *model;
Chris@441 429 if (haveExtents) {
Chris@441 430 model = new RegionModel
Chris@441 431 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 432 } else {
Chris@441 433 model = new RegionModel
Chris@441 434 (modelRate, modelResolution, false);
Chris@441 435 }
Chris@849 436 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 437 out = model;
Chris@441 438 }
Chris@441 439
Chris@558 440 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 441 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 442
Chris@876 443 } else if (binCount == 1 ||
Chris@849 444 (m_descriptors[n]->sampleType ==
Chris@441 445 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 446
Chris@441 447 // Anything that is not a 1D, note, or interval model and that
Chris@441 448 // has only one value per result must be a sparse time value
Chris@441 449 // model.
Chris@441 450
Chris@441 451 // Anything that is not a 1D, note, or interval model and that
Chris@876 452 // has a variable sample rate is treated as a set of sparse
Chris@876 453 // time value models, one per output bin, because we lack a
Chris@441 454 // sparse 3D model.
Chris@320 455
Chris@876 456 // Anything that is not a 1D, note, or interval model and that
Chris@876 457 // has a fixed sample rate but an unknown number of values per
Chris@876 458 // result is also treated as a set of sparse time value models.
Chris@876 459
Chris@876 460 // For sets of sparse time value models, we create a single
Chris@876 461 // model first as the "standard" output and then create models
Chris@876 462 // for bins 1+ in the additional model map (mapping the output
Chris@876 463 // descriptor to a list of models indexed by bin-1). But we
Chris@876 464 // don't create the additional models yet, as this case has to
Chris@876 465 // work even if the number of bins is unknown at this point --
Chris@877 466 // we create an additional model (copying its parameters from
Chris@877 467 // the default one) each time a new bin is encountered.
Chris@876 468
Chris@876 469 if (!haveBinCount || binCount > 1) {
Chris@876 470 m_needAdditionalModels[n] = true;
Chris@876 471 }
Chris@876 472
Chris@320 473 SparseTimeValueModel *model;
Chris@320 474 if (haveExtents) {
Chris@320 475 model = new SparseTimeValueModel
Chris@320 476 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 477 } else {
Chris@320 478 model = new SparseTimeValueModel
Chris@320 479 (modelRate, modelResolution, false);
Chris@320 480 }
Chris@558 481
Chris@558 482 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@849 483 model->setScaleUnits(outputs[m_outputNos[n]].unit.c_str());
Chris@320 484
Chris@849 485 out = model;
Chris@320 486
Chris@558 487 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 488 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 489
Chris@441 490 } else {
Chris@320 491
Chris@441 492 // Anything that is not a 1D, note, or interval model and that
Chris@441 493 // has a fixed sample rate and more than one value per result
Chris@441 494 // must be a dense 3D model.
Chris@320 495
Chris@320 496 EditableDenseThreeDimensionalModel *model =
Chris@320 497 new EditableDenseThreeDimensionalModel
Chris@535 498 (modelRate, modelResolution, binCount,
Chris@535 499 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 500 false);
Chris@320 501
Chris@1429 502 if (!m_descriptors[n]->binNames.empty()) {
Chris@1429 503 std::vector<QString> names;
Chris@1429 504 for (int i = 0; i < (int)m_descriptors[n]->binNames.size(); ++i) {
Chris@1429 505 names.push_back(m_descriptors[n]->binNames[i].c_str());
Chris@1429 506 }
Chris@1429 507 model->setBinNames(names);
Chris@1429 508 }
Chris@320 509
Chris@849 510 out = model;
Chris@558 511
Chris@558 512 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
Chris@849 513 out->setRDFTypeURI(outputSignalTypeURI);
Chris@320 514 }
Chris@333 515
Chris@849 516 if (out) {
Chris@849 517 out->setSourceModel(input);
Chris@849 518 m_outputs.push_back(out);
Chris@849 519 }
Chris@320 520 }
Chris@320 521
Chris@1211 522 void
Chris@1211 523 FeatureExtractionModelTransformer::awaitOutputModels()
Chris@1211 524 {
Chris@1211 525 m_outputMutex.lock();
Chris@1368 526 while (!m_haveOutputs && !m_abandoned) {
Chris@1368 527 m_outputsCondition.wait(&m_outputMutex, 500);
Chris@1211 528 }
Chris@1211 529 m_outputMutex.unlock();
Chris@1211 530 }
Chris@1211 531
Chris@331 532 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 533 {
Chris@1237 534 // Parent class dtor set the abandoned flag and waited for the run
Chris@1237 535 // thread to exit; the run thread owns the plugin, and should have
Chris@1237 536 // destroyed it before exiting (via a call to deinitialise)
Chris@320 537 }
Chris@320 538
Chris@876 539 FeatureExtractionModelTransformer::Models
Chris@876 540 FeatureExtractionModelTransformer::getAdditionalOutputModels()
Chris@876 541 {
Chris@876 542 Models mm;
Chris@876 543 for (AdditionalModelMap::iterator i = m_additionalModels.begin();
Chris@876 544 i != m_additionalModels.end(); ++i) {
Chris@876 545 for (std::map<int, SparseTimeValueModel *>::iterator j =
Chris@876 546 i->second.begin();
Chris@876 547 j != i->second.end(); ++j) {
Chris@876 548 SparseTimeValueModel *m = j->second;
Chris@876 549 if (m) mm.push_back(m);
Chris@876 550 }
Chris@876 551 }
Chris@876 552 return mm;
Chris@876 553 }
Chris@876 554
Chris@877 555 bool
Chris@877 556 FeatureExtractionModelTransformer::willHaveAdditionalOutputModels()
Chris@877 557 {
Chris@877 558 for (std::map<int, bool>::const_iterator i =
Chris@877 559 m_needAdditionalModels.begin();
Chris@877 560 i != m_needAdditionalModels.end(); ++i) {
Chris@877 561 if (i->second) return true;
Chris@877 562 }
Chris@877 563 return false;
Chris@877 564 }
Chris@877 565
Chris@876 566 SparseTimeValueModel *
Chris@876 567 FeatureExtractionModelTransformer::getAdditionalModel(int n, int binNo)
Chris@876 568 {
Chris@893 569 // std::cerr << "getAdditionalModel(" << n << ", " << binNo << ")" << std::endl;
Chris@876 570
Chris@876 571 if (binNo == 0) {
Chris@876 572 std::cerr << "Internal error: binNo == 0 in getAdditionalModel (should be using primary model)" << std::endl;
Chris@1582 573 return nullptr;
Chris@876 574 }
Chris@876 575
Chris@1582 576 if (!m_needAdditionalModels[n]) return nullptr;
Chris@1582 577 if (!isOutput<SparseTimeValueModel>(n)) return nullptr;
Chris@876 578 if (m_additionalModels[n][binNo]) return m_additionalModels[n][binNo];
Chris@876 579
Chris@876 580 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): creating" << std::endl;
Chris@876 581
Chris@876 582 SparseTimeValueModel *baseModel = getConformingOutput<SparseTimeValueModel>(n);
Chris@1582 583 if (!baseModel) return nullptr;
Chris@876 584
Chris@876 585 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): (from " << baseModel << ")" << std::endl;
Chris@876 586
Chris@876 587 SparseTimeValueModel *additional =
Chris@876 588 new SparseTimeValueModel(baseModel->getSampleRate(),
Chris@876 589 baseModel->getResolution(),
Chris@876 590 baseModel->getValueMinimum(),
Chris@876 591 baseModel->getValueMaximum(),
Chris@876 592 false);
Chris@876 593
Chris@876 594 additional->setScaleUnits(baseModel->getScaleUnits());
Chris@876 595 additional->setRDFTypeURI(baseModel->getRDFTypeURI());
Chris@876 596
Chris@876 597 m_additionalModels[n][binNo] = additional;
Chris@876 598 return additional;
Chris@876 599 }
Chris@876 600
Chris@320 601 DenseTimeValueModel *
Chris@350 602 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 603 {
Chris@690 604 // SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
Chris@408 605
Chris@320 606 DenseTimeValueModel *dtvm =
Chris@1429 607 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 608 if (!dtvm) {
Chris@1429 609 SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
Chris@320 610 }
Chris@320 611 return dtvm;
Chris@320 612 }
Chris@320 613
Chris@320 614 void
Chris@331 615 FeatureExtractionModelTransformer::run()
Chris@320 616 {
Chris@1373 617 try {
Chris@1373 618 if (!initialise()) {
Chris@1373 619 abandon();
Chris@1373 620 return;
Chris@1373 621 }
Chris@1373 622 } catch (const std::exception &e) {
Chris@1368 623 abandon();
Chris@1373 624 m_message = e.what();
Chris@1368 625 return;
Chris@1368 626 }
Chris@1211 627
Chris@350 628 DenseTimeValueModel *input = getConformingInput();
Chris@1368 629 if (!input) {
Chris@1368 630 abandon();
Chris@1368 631 return;
Chris@1368 632 }
Chris@320 633
Chris@1368 634 if (m_outputs.empty()) {
Chris@1368 635 abandon();
Chris@1368 636 return;
Chris@1368 637 }
Chris@320 638
Chris@850 639 Transform primaryTransform = m_transforms[0];
Chris@850 640
Chris@497 641 while (!input->isReady() && !m_abandoned) {
Chris@1558 642 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
Chris@497 643 usleep(500000);
Chris@320 644 }
Chris@1558 645 SVDEBUG << "FeatureExtractionModelTransformer::run: Waited, ready = "
Chris@1558 646 << input->isReady() << ", m_abandoned = " << m_abandoned << endl;
Chris@497 647 if (m_abandoned) return;
Chris@320 648
Chris@1040 649 sv_samplerate_t sampleRate = input->getSampleRate();
Chris@320 650
Chris@930 651 int channelCount = input->getChannelCount();
Chris@930 652 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 653 channelCount = 1;
Chris@320 654 }
Chris@320 655
Chris@320 656 float **buffers = new float*[channelCount];
Chris@930 657 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1429 658 buffers[ch] = new float[primaryTransform.getBlockSize() + 2];
Chris@320 659 }
Chris@320 660
Chris@930 661 int stepSize = primaryTransform.getStepSize();
Chris@930 662 int blockSize = primaryTransform.getBlockSize();
Chris@350 663
Chris@320 664 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 665 Vamp::Plugin::FrequencyDomain);
Chris@1558 666
Chris@320 667 std::vector<FFTModel *> fftModels;
Chris@320 668
Chris@320 669 if (frequencyDomain) {
Chris@930 670 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 671 FFTModel *model = new FFTModel
Chris@350 672 (getConformingInput(),
Chris@350 673 channelCount == 1 ? m_input.getChannel() : ch,
Chris@850 674 primaryTransform.getWindowType(),
Chris@350 675 blockSize,
Chris@350 676 stepSize,
Chris@1090 677 blockSize);
Chris@1080 678 if (!model->isOK() || model->getError() != "") {
Chris@1080 679 QString err = model->getError();
Chris@320 680 delete model;
Chris@850 681 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 682 setCompletion(j, 100);
Chris@850 683 }
Chris@387 684 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@1080 685 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer: error is: " + err);
Chris@320 686 }
Chris@320 687 fftModels.push_back(model);
Chris@1080 688 cerr << "created model for channel " << ch << endl;
Chris@320 689 }
Chris@320 690 }
Chris@320 691
Chris@1040 692 sv_frame_t startFrame = m_input.getModel()->getStartFrame();
Chris@1040 693 sv_frame_t endFrame = m_input.getModel()->getEndFrame();
Chris@320 694
Chris@850 695 RealTime contextStartRT = primaryTransform.getStartTime();
Chris@850 696 RealTime contextDurationRT = primaryTransform.getDuration();
Chris@350 697
Chris@1040 698 sv_frame_t contextStart =
Chris@350 699 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 700
Chris@1040 701 sv_frame_t contextDuration =
Chris@350 702 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 703
Chris@320 704 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 705 contextStart = startFrame;
Chris@320 706 }
Chris@320 707
Chris@320 708 if (contextDuration == 0) {
Chris@320 709 contextDuration = endFrame - contextStart;
Chris@320 710 }
Chris@320 711 if (contextStart + contextDuration > endFrame) {
Chris@320 712 contextDuration = endFrame - contextStart;
Chris@320 713 }
Chris@320 714
Chris@1039 715 sv_frame_t blockFrame = contextStart;
Chris@320 716
Chris@320 717 long prevCompletion = 0;
Chris@320 718
Chris@850 719 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 720 setCompletion(j, 0);
Chris@850 721 }
Chris@320 722
Chris@1582 723 float *reals = nullptr;
Chris@1582 724 float *imaginaries = nullptr;
Chris@556 725 if (frequencyDomain) {
Chris@556 726 reals = new float[blockSize/2 + 1];
Chris@556 727 imaginaries = new float[blockSize/2 + 1];
Chris@556 728 }
Chris@556 729
Chris@678 730 QString error = "";
Chris@678 731
Chris@1372 732 try {
Chris@1372 733 while (!m_abandoned) {
Chris@320 734
Chris@1372 735 if (frequencyDomain) {
Chris@1372 736 if (blockFrame - int(blockSize)/2 >
Chris@1372 737 contextStart + contextDuration) break;
Chris@1372 738 } else {
Chris@1372 739 if (blockFrame >=
Chris@1372 740 contextStart + contextDuration) break;
Chris@1372 741 }
Chris@320 742
Chris@1558 743 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558 744 SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@1558 745 << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@1558 746 << blockSize << endl;
Chris@1558 747 #endif
Chris@1558 748
Chris@1372 749 int completion = int
Chris@1372 750 ((((blockFrame - contextStart) / stepSize) * 99) /
Chris@1372 751 (contextDuration / stepSize + 1));
Chris@320 752
Chris@1372 753 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 754
Chris@1372 755 if (frequencyDomain) {
Chris@1372 756 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1372 757 int column = int((blockFrame - startFrame) / stepSize);
Chris@1372 758 if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) {
Chris@1372 759 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 760 buffers[ch][i*2] = reals[i];
Chris@1372 761 buffers[ch][i*2+1] = imaginaries[i];
Chris@1372 762 }
Chris@1372 763 } else {
Chris@1372 764 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 765 buffers[ch][i*2] = 0.f;
Chris@1372 766 buffers[ch][i*2+1] = 0.f;
Chris@1372 767 }
Chris@1558 768 }
Chris@1558 769
Chris@1372 770 error = fftModels[ch]->getError();
Chris@1372 771 if (error != "") {
Chris@1372 772 SVCERR << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << endl;
Chris@1372 773 m_abandoned = true;
Chris@1372 774 m_message = error;
Chris@1372 775 break;
Chris@1008 776 }
Chris@1372 777 }
Chris@1372 778 } else {
Chris@1372 779 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@1372 780 }
Chris@1372 781
Chris@1372 782 if (m_abandoned) break;
Chris@1372 783
Chris@1372 784 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@1558 785 (buffers,
Chris@1558 786 RealTime::frame2RealTime(blockFrame, sampleRate)
Chris@1558 787 .toVampRealTime());
Chris@1558 788
Chris@1372 789 if (m_abandoned) break;
Chris@1372 790
Chris@1372 791 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 792 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 793 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 794 addFeature(j, blockFrame, feature);
Chris@678 795 }
Chris@363 796 }
Chris@1372 797
Chris@1372 798 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@1372 799 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 800 setCompletion(j, completion);
Chris@1372 801 }
Chris@1372 802 prevCompletion = completion;
Chris@1372 803 }
Chris@1372 804
Chris@1372 805 blockFrame += stepSize;
Chris@1372 806
Chris@320 807 }
Chris@320 808
Chris@1372 809 if (!m_abandoned) {
Chris@1372 810 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@497 811
Chris@1372 812 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 813 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 814 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 815 addFeature(j, blockFrame, feature);
Chris@1372 816 }
Chris@850 817 }
Chris@850 818 }
Chris@1372 819 } catch (const std::exception &e) {
Chris@1372 820 SVCERR << "FeatureExtractionModelTransformer::run: Exception caught: "
Chris@1372 821 << e.what() << endl;
Chris@1372 822 m_abandoned = true;
Chris@1372 823 m_message = e.what();
Chris@497 824 }
Chris@320 825
Chris@850 826 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 827 setCompletion(j, 100);
Chris@850 828 }
Chris@320 829
Chris@320 830 if (frequencyDomain) {
Chris@930 831 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 832 delete fftModels[ch];
Chris@320 833 }
Chris@556 834 delete[] reals;
Chris@556 835 delete[] imaginaries;
Chris@320 836 }
Chris@974 837
Chris@974 838 for (int ch = 0; ch < channelCount; ++ch) {
Chris@974 839 delete[] buffers[ch];
Chris@974 840 }
Chris@974 841 delete[] buffers;
Chris@1237 842
Chris@1237 843 deinitialise();
Chris@320 844 }
Chris@320 845
Chris@320 846 void
Chris@363 847 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@1039 848 sv_frame_t startFrame,
Chris@1039 849 sv_frame_t size,
Chris@363 850 float **buffers)
Chris@320 851 {
Chris@1039 852 sv_frame_t offset = 0;
Chris@320 853
Chris@320 854 if (startFrame < 0) {
Chris@363 855 for (int c = 0; c < channelCount; ++c) {
Chris@1039 856 for (sv_frame_t i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 857 buffers[c][i] = 0.0f;
Chris@363 858 }
Chris@320 859 }
Chris@320 860 offset = -startFrame;
Chris@320 861 size -= offset;
Chris@320 862 if (size <= 0) return;
Chris@320 863 startFrame = 0;
Chris@320 864 }
Chris@320 865
Chris@350 866 DenseTimeValueModel *input = getConformingInput();
Chris@350 867 if (!input) return;
Chris@363 868
Chris@1039 869 sv_frame_t got = 0;
Chris@350 870
Chris@363 871 if (channelCount == 1) {
Chris@363 872
Chris@1096 873 auto data = input->getData(m_input.getChannel(), startFrame, size);
Chris@1096 874 got = data.size();
Chris@1096 875
Chris@1096 876 copy(data.begin(), data.end(), buffers[0] + offset);
Chris@363 877
Chris@363 878 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 879 // use mean instead of sum, as plugin input
Chris@363 880 float cc = float(input->getChannelCount());
Chris@1096 881 for (sv_frame_t i = 0; i < got; ++i) {
Chris@363 882 buffers[0][i + offset] /= cc;
Chris@363 883 }
Chris@363 884 }
Chris@363 885
Chris@363 886 } else {
Chris@363 887
Chris@1096 888 auto data = input->getMultiChannelData(0, channelCount-1, startFrame, size);
Chris@1096 889 if (!data.empty()) {
Chris@1096 890 got = data[0].size();
Chris@1096 891 for (int c = 0; in_range_for(data, c); ++c) {
Chris@1096 892 copy(data[c].begin(), data[c].end(), buffers[c] + offset);
Chris@363 893 }
Chris@363 894 }
Chris@363 895 }
Chris@320 896
Chris@320 897 while (got < size) {
Chris@363 898 for (int c = 0; c < channelCount; ++c) {
Chris@363 899 buffers[c][got + offset] = 0.0;
Chris@363 900 }
Chris@320 901 ++got;
Chris@320 902 }
Chris@320 903 }
Chris@320 904
Chris@320 905 void
Chris@850 906 FeatureExtractionModelTransformer::addFeature(int n,
Chris@1039 907 sv_frame_t blockFrame,
Chris@850 908 const Vamp::Plugin::Feature &feature)
Chris@320 909 {
Chris@1040 910 sv_samplerate_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 911
Chris@843 912 // cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
Chris@712 913 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
Chris@712 914 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
Chris@712 915 // << feature.hasDuration << ", duration = " << feature.duration
Chris@843 916 // << endl;
Chris@320 917
Chris@1039 918 sv_frame_t frame = blockFrame;
Chris@320 919
Chris@849 920 if (m_descriptors[n]->sampleType ==
Chris@1429 921 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 922
Chris@1429 923 if (!feature.hasTimestamp) {
Chris@1429 924 SVDEBUG
Chris@1429 925 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@1429 926 << "Feature has variable sample rate but no timestamp!"
Chris@1429 927 << endl;
Chris@1429 928 return;
Chris@1429 929 } else {
Chris@1429 930 frame = RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@1429 931 }
Chris@320 932
Chris@1071 933 // cerr << "variable sample rate: timestamp = " << feature.timestamp
Chris@1071 934 // << " at input rate " << inputRate << " -> " << frame << endl;
Chris@1071 935
Chris@849 936 } else if (m_descriptors[n]->sampleType ==
Chris@1429 937 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 938
Chris@1071 939 sv_samplerate_t rate = m_descriptors[n]->sampleRate;
Chris@1071 940 if (rate <= 0.0) {
Chris@1071 941 rate = inputRate;
Chris@1071 942 }
Chris@1071 943
Chris@779 944 if (!feature.hasTimestamp) {
Chris@849 945 ++m_fixedRateFeatureNos[n];
Chris@779 946 } else {
Chris@779 947 RealTime ts(feature.timestamp.sec, feature.timestamp.nsec);
Chris@1071 948 m_fixedRateFeatureNos[n] = (int)lrint(ts.toDouble() * rate);
Chris@779 949 }
Chris@862 950
Chris@1071 951 // cerr << "m_fixedRateFeatureNo = " << m_fixedRateFeatureNos[n]
Chris@1071 952 // << ", m_descriptor->sampleRate = " << m_descriptors[n]->sampleRate
Chris@862 953 // << ", inputRate = " << inputRate
Chris@862 954 // << " giving frame = ";
Chris@1071 955 frame = lrint((double(m_fixedRateFeatureNos[n]) / rate) * inputRate);
Chris@1071 956 // cerr << frame << endl;
Chris@320 957 }
Chris@862 958
Chris@862 959 if (frame < 0) {
Chris@1264 960 SVDEBUG
Chris@862 961 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@862 962 << "Negative frame counts are not supported (frame = " << frame
Chris@862 963 << " from timestamp " << feature.timestamp
Chris@862 964 << "), dropping feature"
Chris@862 965 << endl;
Chris@862 966 return;
Chris@862 967 }
Chris@862 968
Chris@441 969 // Rather than repeat the complicated tests from the constructor
Chris@441 970 // to determine what sort of model we must be adding the features
Chris@441 971 // to, we instead test what sort of model the constructor decided
Chris@441 972 // to create.
Chris@320 973
Chris@849 974 if (isOutput<SparseOneDimensionalModel>(n)) {
Chris@441 975
Chris@441 976 SparseOneDimensionalModel *model =
Chris@849 977 getConformingOutput<SparseOneDimensionalModel>(n);
Chris@1429 978 if (!model) return;
Chris@350 979
Chris@441 980 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 981 (frame, feature.label.c_str()));
Chris@1429 982
Chris@849 983 } else if (isOutput<SparseTimeValueModel>(n)) {
Chris@320 984
Chris@1429 985 SparseTimeValueModel *model =
Chris@849 986 getConformingOutput<SparseTimeValueModel>(n);
Chris@1429 987 if (!model) return;
Chris@350 988
Chris@930 989 for (int i = 0; i < (int)feature.values.size(); ++i) {
Chris@454 990
Chris@454 991 float value = feature.values[i];
Chris@454 992
Chris@454 993 QString label = feature.label.c_str();
Chris@454 994 if (feature.values.size() > 1) {
Chris@454 995 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 996 }
Chris@454 997
Chris@876 998 SparseTimeValueModel *targetModel = model;
Chris@876 999
Chris@876 1000 if (m_needAdditionalModels[n] && i > 0) {
Chris@876 1001 targetModel = getAdditionalModel(n, i);
Chris@876 1002 if (!targetModel) targetModel = model;
Chris@893 1003 // std::cerr << "adding point to model " << targetModel
Chris@893 1004 // << " for output " << n << " bin " << i << std::endl;
Chris@876 1005 }
Chris@876 1006
Chris@1651 1007 targetModel->add(Event(frame, value, label));
Chris@454 1008 }
Chris@320 1009
Chris@1647 1010 } else if (isOutput<NoteModel>(n) || isOutput<RegionModel>(n)) {
Chris@320 1011
Chris@441 1012 int index = 0;
Chris@441 1013
Chris@441 1014 float value = 0.0;
Chris@930 1015 if ((int)feature.values.size() > index) {
Chris@441 1016 value = feature.values[index++];
Chris@441 1017 }
Chris@320 1018
Chris@1039 1019 sv_frame_t duration = 1;
Chris@441 1020 if (feature.hasDuration) {
Chris@1040 1021 duration = RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 1022 } else {
Chris@1039 1023 if (in_range_for(feature.values, index)) {
Chris@1039 1024 duration = lrintf(feature.values[index++]);
Chris@441 1025 }
Chris@441 1026 }
gyorgyf@786 1027
Chris@1647 1028 if (isOutput<NoteModel>(n)) {
Chris@320 1029
Chris@441 1030 float velocity = 100;
Chris@930 1031 if ((int)feature.values.size() > index) {
Chris@441 1032 velocity = feature.values[index++];
Chris@441 1033 }
Chris@441 1034 if (velocity < 0) velocity = 127;
Chris@441 1035 if (velocity > 127) velocity = 127;
Chris@320 1036
Chris@849 1037 NoteModel *model = getConformingOutput<NoteModel>(n);
Chris@441 1038 if (!model) return;
Chris@1644 1039 model->add(Event(frame, value, // value is pitch
Chris@1644 1040 duration,
Chris@1644 1041 velocity / 127.f,
Chris@1644 1042 feature.label.c_str()));
Chris@441 1043 } else {
gyorgyf@786 1044
Chris@849 1045 RegionModel *model = getConformingOutput<RegionModel>(n);
Chris@454 1046 if (!model) return;
Chris@454 1047
Chris@474 1048 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 1049
Chris@930 1050 for (int i = 0; i < (int)feature.values.size(); ++i) {
Chris@454 1051
Chris@454 1052 float value = feature.values[i];
Chris@454 1053
Chris@454 1054 QString label = feature.label.c_str();
Chris@454 1055 if (feature.values.size() > 1) {
Chris@454 1056 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 1057 }
Chris@454 1058
Chris@1649 1059 model->add(Event(frame,
Chris@1649 1060 value,
Chris@1649 1061 duration,
Chris@1649 1062 label));
Chris@454 1063 }
Chris@454 1064 } else {
Chris@454 1065
Chris@1649 1066 model->add(Event(frame,
Chris@1649 1067 value,
Chris@1649 1068 duration,
Chris@1649 1069 feature.label.c_str()));
Chris@454 1070 }
Chris@441 1071 }
Chris@1429 1072
Chris@849 1073 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) {
Chris@1429 1074
Chris@1429 1075 DenseThreeDimensionalModel::Column values = feature.values;
Chris@1429 1076
Chris@1429 1077 EditableDenseThreeDimensionalModel *model =
Chris@849 1078 getConformingOutput<EditableDenseThreeDimensionalModel>(n);
Chris@1429 1079 if (!model) return;
Chris@320 1080
Chris@889 1081 // cerr << "(note: model resolution = " << model->getResolution() << ")"
Chris@889 1082 // << endl;
Chris@889 1083
Chris@891 1084 if (!feature.hasTimestamp && m_fixedRateFeatureNos[n] >= 0) {
Chris@891 1085 model->setColumn(m_fixedRateFeatureNos[n], values);
Chris@889 1086 } else {
Chris@1039 1087 model->setColumn(int(frame / model->getResolution()), values);
Chris@889 1088 }
Chris@441 1089
Chris@441 1090 } else {
Chris@690 1091 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
Chris@320 1092 }
Chris@320 1093 }
Chris@320 1094
Chris@320 1095 void
Chris@850 1096 FeatureExtractionModelTransformer::setCompletion(int n, int completion)
Chris@320 1097 {
Chris@1558 1098 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558 1099 SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
Chris@1558 1100 << completion << ")" << endl;
Chris@1558 1101 #endif
Chris@320 1102
Chris@849 1103 if (isOutput<SparseOneDimensionalModel>(n)) {
Chris@320 1104
Chris@1429 1105 SparseOneDimensionalModel *model =
Chris@849 1106 getConformingOutput<SparseOneDimensionalModel>(n);
Chris@1429 1107 if (!model) return;
Chris@923 1108 if (model->isAbandoning()) abandon();
Chris@1429 1109 model->setCompletion(completion, true);
Chris@320 1110
Chris@849 1111 } else if (isOutput<SparseTimeValueModel>(n)) {
Chris@320 1112
Chris@1429 1113 SparseTimeValueModel *model =
Chris@849 1114 getConformingOutput<SparseTimeValueModel>(n);
Chris@1429 1115 if (!model) return;
Chris@923 1116 if (model->isAbandoning()) abandon();
Chris@1429 1117 model->setCompletion(completion, true);
Chris@320 1118
Chris@849 1119 } else if (isOutput<NoteModel>(n)) {
Chris@320 1120
Chris@1429 1121 NoteModel *model = getConformingOutput<NoteModel>(n);
Chris@1429 1122 if (!model) return;
Chris@923 1123 if (model->isAbandoning()) abandon();
Chris@1429 1124 model->setCompletion(completion, true);
Chris@1429 1125
Chris@849 1126 } else if (isOutput<RegionModel>(n)) {
Chris@441 1127
Chris@1429 1128 RegionModel *model = getConformingOutput<RegionModel>(n);
Chris@1429 1129 if (!model) return;
Chris@923 1130 if (model->isAbandoning()) abandon();
Chris@1429 1131 model->setCompletion(completion, true);
Chris@441 1132
Chris@849 1133 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) {
Chris@320 1134
Chris@1429 1135 EditableDenseThreeDimensionalModel *model =
Chris@849 1136 getConformingOutput<EditableDenseThreeDimensionalModel>(n);
Chris@1429 1137 if (!model) return;
Chris@923 1138 if (model->isAbandoning()) abandon();
Chris@1429 1139 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 1140 }
Chris@320 1141 }
Chris@320 1142