annotate transform/FeatureExtractionModelTransformer.cpp @ 1739:565575463752 by-id

Some work on models and transformers
author Chris Cannam
date Wed, 26 Jun 2019 14:59:09 +0100
parents 5b7b01da430a
children fe3f7f8df3a3
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@1225 19
Chris@320 20 #include "plugin/PluginXml.h"
Chris@475 21 #include <vamp-hostsdk/Plugin.h>
Chris@320 22
Chris@320 23 #include "data/model/Model.h"
Chris@320 24 #include "base/Window.h"
Chris@387 25 #include "base/Exceptions.h"
Chris@320 26 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 27 #include "data/model/SparseTimeValueModel.h"
Chris@320 28 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 29 #include "data/model/DenseTimeValueModel.h"
Chris@320 30 #include "data/model/NoteModel.h"
Chris@441 31 #include "data/model/RegionModel.h"
Chris@320 32 #include "data/model/FFTModel.h"
Chris@320 33 #include "data/model/WaveFileModel.h"
Chris@558 34 #include "rdf/PluginRDFDescription.h"
Chris@320 35
Chris@350 36 #include "TransformFactory.h"
Chris@350 37
Chris@320 38 #include <iostream>
Chris@320 39
Chris@859 40 #include <QSettings>
Chris@859 41
Chris@1558 42 //#define DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 1
Chris@1558 43
Chris@350 44 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 45 const Transform &transform) :
Chris@350 46 ModelTransformer(in, transform),
Chris@1582 47 m_plugin(nullptr),
Chris@1211 48 m_haveOutputs(false)
Chris@320 49 {
Chris@1080 50 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@849 51 }
Chris@849 52
Chris@849 53 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 54 const Transforms &transforms) :
Chris@849 55 ModelTransformer(in, transforms),
Chris@1582 56 m_plugin(nullptr),
Chris@1211 57 m_haveOutputs(false)
Chris@849 58 {
Chris@1080 59 if (m_transforms.empty()) {
Chris@1080 60 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s)" << endl;
Chris@1080 61 } else {
Chris@1080 62 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s), first has plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@1080 63 }
Chris@849 64 }
Chris@849 65
Chris@849 66 static bool
Chris@849 67 areTransformsSimilar(const Transform &t1, const Transform &t2)
Chris@849 68 {
Chris@849 69 Transform t2o(t2);
Chris@849 70 t2o.setOutput(t1.getOutput());
Chris@849 71 return t1 == t2o;
Chris@849 72 }
Chris@849 73
Chris@849 74 bool
Chris@849 75 FeatureExtractionModelTransformer::initialise()
Chris@849 76 {
Chris@1237 77 // This is (now) called from the run thread. The plugin is
Chris@1237 78 // constructed, initialised, used, and destroyed all from a single
Chris@1237 79 // thread.
Chris@1237 80
Chris@849 81 // All transforms must use the same plugin, parameters, and
Chris@849 82 // inputs: they can differ only in choice of plugin output. So we
Chris@849 83 // initialise based purely on the first transform in the list (but
Chris@849 84 // first check that they are actually similar as promised)
Chris@849 85
Chris@1739 86 for (int j = 1; in_range_for(m_transforms, j); ++j) {
Chris@849 87 if (!areTransformsSimilar(m_transforms[0], m_transforms[j])) {
Chris@849 88 m_message = tr("Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output");
Chris@1368 89 SVCERR << m_message << endl;
Chris@849 90 return false;
Chris@849 91 }
Chris@849 92 }
Chris@849 93
Chris@849 94 Transform primaryTransform = m_transforms[0];
Chris@849 95
Chris@849 96 QString pluginId = primaryTransform.getPluginIdentifier();
Chris@320 97
Chris@1226 98 FeatureExtractionPluginFactory *factory =
Chris@1226 99 FeatureExtractionPluginFactory::instance();
Chris@320 100
Chris@320 101 if (!factory) {
Chris@361 102 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@1368 103 SVCERR << m_message << endl;
Chris@1429 104 return false;
Chris@320 105 }
Chris@320 106
Chris@1739 107 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@350 108 if (!input) {
Chris@361 109 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@1368 110 SVCERR << m_message << endl;
Chris@849 111 return false;
Chris@350 112 }
Chris@320 113
Chris@1264 114 SVDEBUG << "FeatureExtractionModelTransformer: Instantiating plugin for transform in thread "
Chris@1264 115 << QThread::currentThreadId() << endl;
Chris@1211 116
Chris@1040 117 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 118 if (!m_plugin) {
Chris@361 119 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@1368 120 SVCERR << m_message << endl;
Chris@1429 121 return false;
Chris@320 122 }
Chris@320 123
Chris@350 124 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@849 125 (primaryTransform, m_plugin);
Chris@1368 126
Chris@350 127 TransformFactory::getInstance()->setPluginParameters
Chris@849 128 (primaryTransform, m_plugin);
Chris@1368 129
Chris@930 130 int channelCount = input->getChannelCount();
Chris@930 131 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 132 channelCount = 1;
Chris@320 133 }
Chris@930 134 if ((int)m_plugin->getMinChannelCount() > channelCount) {
Chris@361 135 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 136 .arg(pluginId)
Chris@361 137 .arg(m_plugin->getMinChannelCount())
Chris@361 138 .arg(m_plugin->getMaxChannelCount())
Chris@361 139 .arg(input->getChannelCount());
Chris@1368 140 SVCERR << m_message << endl;
Chris@1429 141 return false;
Chris@320 142 }
Chris@1374 143
Chris@1374 144 int step = primaryTransform.getStepSize();
Chris@1374 145 int block = primaryTransform.getBlockSize();
Chris@1368 146
Chris@690 147 SVDEBUG << "Initialising feature extraction plugin with channels = "
Chris@1374 148 << channelCount << ", step = " << step
Chris@1374 149 << ", block = " << block << endl;
Chris@320 150
Chris@1374 151 if (!m_plugin->initialise(channelCount, step, block)) {
Chris@1374 152
Chris@1374 153 int preferredStep = int(m_plugin->getPreferredStepSize());
Chris@1374 154 int preferredBlock = int(m_plugin->getPreferredBlockSize());
Chris@1264 155
Chris@1374 156 if (step != preferredStep || block != preferredBlock) {
Chris@361 157
Chris@1374 158 SVDEBUG << "Initialisation failed, trying again with preferred step = "
Chris@1374 159 << preferredStep << ", block = " << preferredBlock << endl;
Chris@361 160
Chris@1739 161 if (!m_plugin->initialise(channelCount,
Chris@1739 162 preferredStep,
Chris@1739 163 preferredBlock)) {
Chris@361 164
Chris@1264 165 SVDEBUG << "Initialisation failed again" << endl;
Chris@1264 166
Chris@361 167 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 168 SVCERR << m_message << endl;
Chris@849 169 return false;
Chris@361 170
Chris@361 171 } else {
Chris@1264 172
Chris@1264 173 SVDEBUG << "Initialisation succeeded this time" << endl;
Chris@1374 174
Chris@1374 175 // Set these values into the primary transform in the list
Chris@1374 176 m_transforms[0].setStepSize(preferredStep);
Chris@1374 177 m_transforms[0].setBlockSize(preferredBlock);
Chris@1264 178
Chris@361 179 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 180 .arg(pluginId)
Chris@1374 181 .arg(step)
Chris@1374 182 .arg(block)
Chris@1374 183 .arg(preferredStep)
Chris@1374 184 .arg(preferredBlock);
Chris@1368 185 SVCERR << m_message << endl;
Chris@361 186 }
Chris@361 187
Chris@361 188 } else {
Chris@361 189
Chris@1374 190 SVDEBUG << "Initialisation failed (with step = " << step
Chris@1374 191 << " and block = " << block
Chris@1374 192 << ", both matching the plugin's preference)" << endl;
Chris@1264 193
Chris@361 194 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 195 SVCERR << m_message << endl;
Chris@849 196 return false;
Chris@361 197 }
Chris@1264 198 } else {
Chris@1264 199 SVDEBUG << "Initialisation succeeded" << endl;
Chris@320 200 }
Chris@320 201
Chris@849 202 if (primaryTransform.getPluginVersion() != "") {
Chris@366 203 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@849 204 if (pv != primaryTransform.getPluginVersion()) {
Chris@366 205 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@849 206 .arg(primaryTransform.getPluginVersion())
Chris@366 207 .arg(pluginId)
Chris@366 208 .arg(pv);
Chris@366 209 if (m_message != "") {
Chris@366 210 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 211 } else {
Chris@366 212 m_message = vm;
Chris@366 213 }
Chris@1368 214 SVCERR << m_message << endl;
Chris@366 215 }
Chris@366 216 }
Chris@366 217
Chris@320 218 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 219
Chris@320 220 if (outputs.empty()) {
Chris@361 221 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@1368 222 SVCERR << m_message << endl;
Chris@1429 223 return false;
Chris@320 224 }
Chris@320 225
Chris@1739 226 for (int j = 0; in_range_for(m_transforms, j); ++j) {
Chris@849 227
Chris@1739 228 for (int i = 0; in_range_for(outputs, i); ++i) {
Chris@1739 229
Chris@849 230 if (m_transforms[j].getOutput() == "" ||
Chris@1739 231 outputs[i].identifier ==
Chris@1739 232 m_transforms[j].getOutput().toStdString()) {
Chris@1739 233
Chris@849 234 m_outputNos.push_back(i);
Chris@1739 235 m_descriptors.push_back(outputs[i]);
Chris@849 236 m_fixedRateFeatureNos.push_back(-1); // we increment before use
Chris@849 237 break;
Chris@849 238 }
Chris@849 239 }
Chris@849 240
Chris@1739 241 if (!in_range_for(m_descriptors, j)) {
Chris@849 242 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@849 243 .arg(pluginId)
Chris@849 244 .arg(m_transforms[j].getOutput());
Chris@1368 245 SVCERR << m_message << endl;
Chris@849 246 return false;
Chris@849 247 }
Chris@320 248 }
Chris@320 249
Chris@1739 250 for (int j = 0; in_range_for(m_transforms, j); ++j) {
Chris@876 251 createOutputModels(j);
Chris@849 252 }
Chris@849 253
Chris@1211 254 m_outputMutex.lock();
Chris@1211 255 m_haveOutputs = true;
Chris@1211 256 m_outputsCondition.wakeAll();
Chris@1211 257 m_outputMutex.unlock();
Chris@1211 258
Chris@849 259 return true;
Chris@558 260 }
Chris@558 261
Chris@558 262 void
Chris@1237 263 FeatureExtractionModelTransformer::deinitialise()
Chris@1237 264 {
Chris@1264 265 SVDEBUG << "FeatureExtractionModelTransformer: deleting plugin for transform in thread "
Chris@1264 266 << QThread::currentThreadId() << endl;
Chris@1372 267
Chris@1372 268 try {
Chris@1372 269 delete m_plugin;
Chris@1372 270 } catch (const std::exception &e) {
Chris@1372 271 // A destructor shouldn't throw an exception. But at one point
Chris@1372 272 // (now fixed) our plugin stub destructor could have
Chris@1372 273 // accidentally done so, so just in case:
Chris@1372 274 SVCERR << "FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl;
Chris@1372 275 m_message = e.what();
Chris@1372 276 }
Chris@1582 277 m_plugin = nullptr;
Chris@1739 278
Chris@1739 279 m_descriptors.clear();
Chris@1237 280 }
Chris@1237 281
Chris@1237 282 void
Chris@876 283 FeatureExtractionModelTransformer::createOutputModels(int n)
Chris@558 284 {
Chris@1739 285 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@1739 286 if (!input) return;
Chris@712 287
Chris@849 288 PluginRDFDescription description(m_transforms[n].getPluginIdentifier());
Chris@849 289 QString outputId = m_transforms[n].getOutput();
Chris@558 290
Chris@320 291 int binCount = 1;
Chris@320 292 float minValue = 0.0, maxValue = 0.0;
Chris@320 293 bool haveExtents = false;
Chris@1739 294 bool haveBinCount = m_descriptors[n].hasFixedBinCount;
Chris@876 295
Chris@876 296 if (haveBinCount) {
Chris@1739 297 binCount = (int)m_descriptors[n].binCount;
Chris@320 298 }
Chris@320 299
Chris@876 300 m_needAdditionalModels[n] = false;
Chris@876 301
Chris@1739 302 if (binCount > 0 && m_descriptors[n].hasKnownExtents) {
Chris@1739 303 minValue = m_descriptors[n].minValue;
Chris@1739 304 maxValue = m_descriptors[n].maxValue;
Chris@320 305 haveExtents = true;
Chris@320 306 }
Chris@320 307
Chris@1040 308 sv_samplerate_t modelRate = input->getSampleRate();
Chris@1254 309 sv_samplerate_t outputRate = modelRate;
Chris@930 310 int modelResolution = 1;
Chris@712 311
Chris@1739 312 if (m_descriptors[n].sampleType !=
Chris@785 313 Vamp::Plugin::OutputDescriptor::OneSamplePerStep) {
Chris@1254 314
Chris@1739 315 outputRate = m_descriptors[n].sampleRate;
Chris@1254 316
Chris@1254 317 //!!! SV doesn't actually support display of models that have
Chris@1254 318 //!!! different underlying rates together -- so we always set
Chris@1254 319 //!!! the model rate to be the input model's rate, and adjust
Chris@1254 320 //!!! the resolution appropriately. We can't properly display
Chris@1254 321 //!!! data with a higher resolution than the base model at all
Chris@1254 322 if (outputRate > input->getSampleRate()) {
Chris@1264 323 SVDEBUG << "WARNING: plugin reports output sample rate as "
Chris@1264 324 << outputRate
Chris@1264 325 << " (can't display features with finer resolution than the input rate of "
Chris@1264 326 << modelRate << ")" << endl;
Chris@1254 327 outputRate = modelRate;
Chris@785 328 }
Chris@785 329 }
Chris@785 330
Chris@1739 331 switch (m_descriptors[n].sampleType) {
Chris@320 332
Chris@320 333 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@1429 334 if (outputRate != 0.0) {
Chris@1429 335 modelResolution = int(round(modelRate / outputRate));
Chris@1429 336 }
Chris@1429 337 break;
Chris@320 338
Chris@320 339 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@1429 340 modelResolution = m_transforms[n].getStepSize();
Chris@1429 341 break;
Chris@320 342
Chris@320 343 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@1254 344 if (outputRate <= 0.0) {
Chris@1739 345 SVDEBUG << "WARNING: Fixed sample-rate plugin reports invalid sample rate " << m_descriptors[n].sampleRate << "; defaulting to input rate of " << input->getSampleRate() << endl;
Chris@1071 346 modelResolution = 1;
Chris@451 347 } else {
Chris@1254 348 modelResolution = int(round(modelRate / outputRate));
Chris@1254 349 // cerr << "modelRate = " << modelRate << ", descriptor rate = " << outputRate << ", modelResolution = " << modelResolution << endl;
Chris@451 350 }
Chris@1429 351 break;
Chris@320 352 }
Chris@320 353
Chris@441 354 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 355
Chris@1739 356 std::shared_ptr<Model> out;
Chris@849 357
Chris@441 358 if (binCount == 0 &&
Chris@1739 359 (preDurationPlugin || !m_descriptors[n].hasDuration)) {
Chris@320 360
Chris@445 361 // Anything with no value and no duration is an instant
Chris@445 362
Chris@1739 363 out = std::make_shared<SparseOneDimensionalModel>
Chris@1739 364 (modelRate, modelResolution, false);
Chris@1739 365
Chris@558 366 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 367 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 368
Chris@441 369 } else if ((preDurationPlugin && binCount > 1 &&
Chris@1739 370 (m_descriptors[n].sampleType ==
Chris@441 371 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@1739 372 (!preDurationPlugin && m_descriptors[n].hasDuration)) {
Chris@441 373
Chris@441 374 // For plugins using the old v1 API without explicit duration,
Chris@441 375 // we treat anything that has multiple bins (i.e. that has the
Chris@441 376 // potential to have value and duration) and a variable sample
Chris@441 377 // rate as a note model, taking its values as pitch, duration
Chris@441 378 // and velocity (if present) respectively. This is the same
Chris@441 379 // behaviour as always applied by SV to these plugins in the
Chris@441 380 // past.
Chris@441 381
Chris@441 382 // For plugins with the newer API, we treat anything with
Chris@441 383 // duration as either a note model with pitch and velocity, or
Chris@441 384 // a region model.
Chris@441 385
Chris@441 386 // How do we know whether it's an interval or note model?
Chris@441 387 // What's the essential difference? Is a note model any
Chris@441 388 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 389 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 390 // note model always have velocity? This is a good question
Chris@441 391 // to be addressed by accompanying RDF, but for the moment we
Chris@441 392 // will do the following...
Chris@441 393
Chris@441 394 bool isNoteModel = false;
Chris@441 395
Chris@441 396 // Regions have only value (and duration -- we can't extract a
Chris@441 397 // region model from an old-style plugin that doesn't support
Chris@441 398 // duration)
Chris@441 399 if (binCount > 1) isNoteModel = true;
Chris@441 400
Chris@595 401 // Regions do not have units of Hz or MIDI things (a sweeping
Chris@595 402 // assumption!)
Chris@1739 403 if (m_descriptors[n].unit == "Hz" ||
Chris@1739 404 m_descriptors[n].unit.find("MIDI") != std::string::npos ||
Chris@1739 405 m_descriptors[n].unit.find("midi") != std::string::npos) {
Chris@595 406 isNoteModel = true;
Chris@595 407 }
Chris@441 408
Chris@441 409 // If we had a "sparse 3D model", we would have the additional
Chris@441 410 // problem of determining whether to use that here (if bin
Chris@441 411 // count > 1). But we don't.
Chris@441 412
Chris@859 413 QSettings settings;
Chris@859 414
Chris@1647 415 if (isNoteModel) {
Chris@441 416
Chris@441 417 NoteModel *model;
Chris@441 418 if (haveExtents) {
Chris@859 419 model = new NoteModel
Chris@859 420 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 421 } else {
Chris@859 422 model = new NoteModel
Chris@859 423 (modelRate, modelResolution, false);
gyorgyf@786 424 }
Chris@1739 425 model->setScaleUnits(m_descriptors[n].unit.c_str());
Chris@1739 426 out.reset(model);
gyorgyf@786 427
Chris@441 428 } else {
Chris@441 429
Chris@441 430 RegionModel *model;
Chris@441 431 if (haveExtents) {
Chris@441 432 model = new RegionModel
Chris@441 433 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 434 } else {
Chris@441 435 model = new RegionModel
Chris@441 436 (modelRate, modelResolution, false);
Chris@441 437 }
Chris@1739 438 model->setScaleUnits(m_descriptors[n].unit.c_str());
Chris@1739 439 out.reset(model);
Chris@441 440 }
Chris@441 441
Chris@558 442 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 443 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 444
Chris@876 445 } else if (binCount == 1 ||
Chris@1739 446 (m_descriptors[n].sampleType ==
Chris@441 447 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 448
Chris@441 449 // Anything that is not a 1D, note, or interval model and that
Chris@441 450 // has only one value per result must be a sparse time value
Chris@441 451 // model.
Chris@441 452
Chris@441 453 // Anything that is not a 1D, note, or interval model and that
Chris@876 454 // has a variable sample rate is treated as a set of sparse
Chris@876 455 // time value models, one per output bin, because we lack a
Chris@441 456 // sparse 3D model.
Chris@320 457
Chris@876 458 // Anything that is not a 1D, note, or interval model and that
Chris@876 459 // has a fixed sample rate but an unknown number of values per
Chris@876 460 // result is also treated as a set of sparse time value models.
Chris@876 461
Chris@876 462 // For sets of sparse time value models, we create a single
Chris@876 463 // model first as the "standard" output and then create models
Chris@876 464 // for bins 1+ in the additional model map (mapping the output
Chris@876 465 // descriptor to a list of models indexed by bin-1). But we
Chris@876 466 // don't create the additional models yet, as this case has to
Chris@876 467 // work even if the number of bins is unknown at this point --
Chris@877 468 // we create an additional model (copying its parameters from
Chris@877 469 // the default one) each time a new bin is encountered.
Chris@876 470
Chris@876 471 if (!haveBinCount || binCount > 1) {
Chris@876 472 m_needAdditionalModels[n] = true;
Chris@876 473 }
Chris@876 474
Chris@320 475 SparseTimeValueModel *model;
Chris@320 476 if (haveExtents) {
Chris@320 477 model = new SparseTimeValueModel
Chris@320 478 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 479 } else {
Chris@320 480 model = new SparseTimeValueModel
Chris@320 481 (modelRate, modelResolution, false);
Chris@320 482 }
Chris@558 483
Chris@558 484 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@849 485 model->setScaleUnits(outputs[m_outputNos[n]].unit.c_str());
Chris@320 486
Chris@1739 487 out.reset(model);
Chris@320 488
Chris@558 489 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 490 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 491
Chris@441 492 } else {
Chris@320 493
Chris@441 494 // Anything that is not a 1D, note, or interval model and that
Chris@441 495 // has a fixed sample rate and more than one value per result
Chris@441 496 // must be a dense 3D model.
Chris@320 497
Chris@320 498 EditableDenseThreeDimensionalModel *model =
Chris@320 499 new EditableDenseThreeDimensionalModel
Chris@535 500 (modelRate, modelResolution, binCount,
Chris@535 501 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 502 false);
Chris@320 503
Chris@1739 504 if (!m_descriptors[n].binNames.empty()) {
Chris@1429 505 std::vector<QString> names;
Chris@1739 506 for (int i = 0; i < (int)m_descriptors[n].binNames.size(); ++i) {
Chris@1739 507 names.push_back(m_descriptors[n].binNames[i].c_str());
Chris@1429 508 }
Chris@1429 509 model->setBinNames(names);
Chris@1429 510 }
Chris@320 511
Chris@1739 512 out.reset(model);
Chris@558 513
Chris@558 514 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
Chris@849 515 out->setRDFTypeURI(outputSignalTypeURI);
Chris@320 516 }
Chris@333 517
Chris@849 518 if (out) {
Chris@1739 519 out->setSourceModel(getInputModel());
Chris@1739 520 ModelById::add(out);
Chris@1739 521 m_outputs.push_back(out->getId());
Chris@849 522 }
Chris@320 523 }
Chris@320 524
Chris@1211 525 void
Chris@1211 526 FeatureExtractionModelTransformer::awaitOutputModels()
Chris@1211 527 {
Chris@1211 528 m_outputMutex.lock();
Chris@1368 529 while (!m_haveOutputs && !m_abandoned) {
Chris@1368 530 m_outputsCondition.wait(&m_outputMutex, 500);
Chris@1211 531 }
Chris@1211 532 m_outputMutex.unlock();
Chris@1211 533 }
Chris@1211 534
Chris@331 535 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 536 {
Chris@1237 537 // Parent class dtor set the abandoned flag and waited for the run
Chris@1237 538 // thread to exit; the run thread owns the plugin, and should have
Chris@1237 539 // destroyed it before exiting (via a call to deinitialise)
Chris@320 540 }
Chris@320 541
Chris@876 542 FeatureExtractionModelTransformer::Models
Chris@876 543 FeatureExtractionModelTransformer::getAdditionalOutputModels()
Chris@876 544 {
Chris@876 545 Models mm;
Chris@1739 546 for (auto mp : m_additionalModels) {
Chris@1739 547 for (auto m: mp.second) {
Chris@1739 548 mm.push_back(m.second);
Chris@876 549 }
Chris@876 550 }
Chris@876 551 return mm;
Chris@876 552 }
Chris@876 553
Chris@877 554 bool
Chris@877 555 FeatureExtractionModelTransformer::willHaveAdditionalOutputModels()
Chris@877 556 {
Chris@1739 557 for (auto p : m_needAdditionalModels) {
Chris@1739 558 if (p.second) return true;
Chris@877 559 }
Chris@877 560 return false;
Chris@877 561 }
Chris@877 562
Chris@1739 563 ModelId
Chris@876 564 FeatureExtractionModelTransformer::getAdditionalModel(int n, int binNo)
Chris@876 565 {
Chris@876 566 if (binNo == 0) {
Chris@1739 567 SVCERR << "Internal error: binNo == 0 in getAdditionalModel (should be using primary model, not calling getAdditionalModel)" << endl;
Chris@1739 568 return {};
Chris@876 569 }
Chris@876 570
Chris@1739 571 if (!in_range_for(m_outputs, n)) {
Chris@1739 572 SVCERR << "getAdditionalModel: Output " << n << " out of range" << endl;
Chris@1739 573 return {};
Chris@1739 574 }
Chris@876 575
Chris@1739 576 if (!in_range_for(m_needAdditionalModels, n) ||
Chris@1739 577 !m_needAdditionalModels[n]) {
Chris@1739 578 return {};
Chris@1739 579 }
Chris@1739 580
Chris@1739 581 if (!m_additionalModels[n][binNo].isNone()) {
Chris@1739 582 return m_additionalModels[n][binNo];
Chris@1739 583 }
Chris@876 584
Chris@1739 585 SVDEBUG << "getAdditionalModel(" << n << ", " << binNo
Chris@1739 586 << "): creating" << endl;
Chris@876 587
Chris@1739 588 auto baseModel = ModelById::getAs<SparseTimeValueModel>(m_outputs[n]);
Chris@1739 589 if (!baseModel) {
Chris@1739 590 SVCERR << "getAdditionalModel: Output model not conformable, or has vanished" << endl;
Chris@1739 591 return {};
Chris@1739 592 }
Chris@1739 593
Chris@1739 594 SVDEBUG << "getAdditionalModel(" << n << ", " << binNo
Chris@1739 595 << "): (from " << baseModel << ")" << endl;
Chris@876 596
Chris@876 597 SparseTimeValueModel *additional =
Chris@876 598 new SparseTimeValueModel(baseModel->getSampleRate(),
Chris@876 599 baseModel->getResolution(),
Chris@876 600 baseModel->getValueMinimum(),
Chris@876 601 baseModel->getValueMaximum(),
Chris@876 602 false);
Chris@876 603
Chris@876 604 additional->setScaleUnits(baseModel->getScaleUnits());
Chris@876 605 additional->setRDFTypeURI(baseModel->getRDFTypeURI());
Chris@876 606
Chris@1739 607 ModelId additionalId = additional->getId();
Chris@1739 608 ModelById::add(std::shared_ptr<SparseTimeValueModel>(additional));
Chris@1739 609 m_additionalModels[n][binNo] = additionalId;
Chris@1739 610 return additionalId;
Chris@320 611 }
Chris@320 612
Chris@320 613 void
Chris@331 614 FeatureExtractionModelTransformer::run()
Chris@320 615 {
Chris@1373 616 try {
Chris@1373 617 if (!initialise()) {
Chris@1373 618 abandon();
Chris@1373 619 return;
Chris@1373 620 }
Chris@1373 621 } catch (const std::exception &e) {
Chris@1368 622 abandon();
Chris@1373 623 m_message = e.what();
Chris@1368 624 return;
Chris@1368 625 }
Chris@320 626
Chris@1368 627 if (m_outputs.empty()) {
Chris@1368 628 abandon();
Chris@1368 629 return;
Chris@1368 630 }
Chris@320 631
Chris@850 632 Transform primaryTransform = m_transforms[0];
Chris@850 633
Chris@1739 634 bool ready = false;
Chris@1739 635 while (!ready && !m_abandoned) {
Chris@1739 636 { // scope so as to release input shared_ptr before sleeping
Chris@1739 637 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@1739 638 if (!input) {
Chris@1739 639 abandon();
Chris@1739 640 return;
Chris@1739 641 }
Chris@1739 642 ready = input->isReady();
Chris@1739 643 }
Chris@1739 644 if (!ready) {
Chris@1739 645 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
Chris@1739 646 usleep(500000);
Chris@1739 647 }
Chris@320 648 }
Chris@497 649 if (m_abandoned) return;
Chris@320 650
Chris@1739 651 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@1739 652 if (!input) {
Chris@1739 653 SVCERR << "FeatureExtractionModelTransformer::run: Input model not (no longer?) available, abandoning" << endl;
Chris@1739 654 abandon();
Chris@1739 655 return;
Chris@1739 656 }
Chris@1739 657
Chris@1040 658 sv_samplerate_t sampleRate = input->getSampleRate();
Chris@320 659
Chris@930 660 int channelCount = input->getChannelCount();
Chris@930 661 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 662 channelCount = 1;
Chris@320 663 }
Chris@320 664
Chris@320 665 float **buffers = new float*[channelCount];
Chris@930 666 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1429 667 buffers[ch] = new float[primaryTransform.getBlockSize() + 2];
Chris@320 668 }
Chris@320 669
Chris@930 670 int stepSize = primaryTransform.getStepSize();
Chris@930 671 int blockSize = primaryTransform.getBlockSize();
Chris@350 672
Chris@320 673 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 674 Vamp::Plugin::FrequencyDomain);
Chris@1558 675
Chris@320 676 std::vector<FFTModel *> fftModels;
Chris@320 677
Chris@320 678 if (frequencyDomain) {
Chris@930 679 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 680 FFTModel *model = new FFTModel
Chris@1739 681 //!!! (input->getId(),
Chris@1739 682 (nullptr,
Chris@1739 683 channelCount == 1 ? m_input.getChannel() : ch,
Chris@1739 684 primaryTransform.getWindowType(),
Chris@1739 685 blockSize,
Chris@1739 686 stepSize,
Chris@1739 687 blockSize);
Chris@1080 688 if (!model->isOK() || model->getError() != "") {
Chris@1080 689 QString err = model->getError();
Chris@320 690 delete model;
Chris@1739 691 for (int j = 0; in_range_for(m_outputNos, j); ++j) {
Chris@850 692 setCompletion(j, 100);
Chris@850 693 }
Chris@387 694 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@1080 695 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer: error is: " + err);
Chris@320 696 }
Chris@320 697 fftModels.push_back(model);
Chris@320 698 }
Chris@320 699 }
Chris@320 700
Chris@1739 701 sv_frame_t startFrame = input->getStartFrame();
Chris@1739 702 sv_frame_t endFrame = input->getEndFrame();
Chris@320 703
Chris@850 704 RealTime contextStartRT = primaryTransform.getStartTime();
Chris@850 705 RealTime contextDurationRT = primaryTransform.getDuration();
Chris@350 706
Chris@1040 707 sv_frame_t contextStart =
Chris@350 708 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 709
Chris@1040 710 sv_frame_t contextDuration =
Chris@350 711 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 712
Chris@320 713 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 714 contextStart = startFrame;
Chris@320 715 }
Chris@320 716
Chris@320 717 if (contextDuration == 0) {
Chris@320 718 contextDuration = endFrame - contextStart;
Chris@320 719 }
Chris@320 720 if (contextStart + contextDuration > endFrame) {
Chris@320 721 contextDuration = endFrame - contextStart;
Chris@320 722 }
Chris@320 723
Chris@1039 724 sv_frame_t blockFrame = contextStart;
Chris@320 725
Chris@320 726 long prevCompletion = 0;
Chris@320 727
Chris@1739 728 for (int j = 0; in_range_for(m_outputNos, j); ++j) {
Chris@850 729 setCompletion(j, 0);
Chris@850 730 }
Chris@320 731
Chris@1582 732 float *reals = nullptr;
Chris@1582 733 float *imaginaries = nullptr;
Chris@556 734 if (frequencyDomain) {
Chris@556 735 reals = new float[blockSize/2 + 1];
Chris@556 736 imaginaries = new float[blockSize/2 + 1];
Chris@556 737 }
Chris@556 738
Chris@678 739 QString error = "";
Chris@678 740
Chris@1372 741 try {
Chris@1372 742 while (!m_abandoned) {
Chris@320 743
Chris@1372 744 if (frequencyDomain) {
Chris@1372 745 if (blockFrame - int(blockSize)/2 >
Chris@1739 746 contextStart + contextDuration) {
Chris@1739 747 break;
Chris@1739 748 }
Chris@1372 749 } else {
Chris@1372 750 if (blockFrame >=
Chris@1739 751 contextStart + contextDuration) {
Chris@1739 752 break;
Chris@1739 753 }
Chris@1372 754 }
Chris@320 755
Chris@1558 756 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558 757 SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@1558 758 << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@1558 759 << blockSize << endl;
Chris@1558 760 #endif
Chris@1558 761
Chris@1372 762 int completion = int
Chris@1372 763 ((((blockFrame - contextStart) / stepSize) * 99) /
Chris@1372 764 (contextDuration / stepSize + 1));
Chris@320 765
Chris@1739 766 // channelCount is either input->channelCount or 1
Chris@320 767
Chris@1372 768 if (frequencyDomain) {
Chris@1372 769 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1372 770 int column = int((blockFrame - startFrame) / stepSize);
Chris@1372 771 if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) {
Chris@1372 772 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 773 buffers[ch][i*2] = reals[i];
Chris@1372 774 buffers[ch][i*2+1] = imaginaries[i];
Chris@1372 775 }
Chris@1372 776 } else {
Chris@1372 777 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 778 buffers[ch][i*2] = 0.f;
Chris@1372 779 buffers[ch][i*2+1] = 0.f;
Chris@1372 780 }
Chris@1558 781 }
Chris@1558 782
Chris@1372 783 error = fftModels[ch]->getError();
Chris@1372 784 if (error != "") {
Chris@1372 785 SVCERR << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << endl;
Chris@1372 786 m_abandoned = true;
Chris@1372 787 m_message = error;
Chris@1372 788 break;
Chris@1008 789 }
Chris@1372 790 }
Chris@1372 791 } else {
Chris@1372 792 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@1372 793 }
Chris@1372 794
Chris@1372 795 if (m_abandoned) break;
Chris@1372 796
Chris@1372 797 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@1558 798 (buffers,
Chris@1558 799 RealTime::frame2RealTime(blockFrame, sampleRate)
Chris@1558 800 .toVampRealTime());
Chris@1558 801
Chris@1372 802 if (m_abandoned) break;
Chris@1372 803
Chris@1372 804 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 805 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 806 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 807 addFeature(j, blockFrame, feature);
Chris@678 808 }
Chris@363 809 }
Chris@1372 810
Chris@1372 811 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@1372 812 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 813 setCompletion(j, completion);
Chris@1372 814 }
Chris@1372 815 prevCompletion = completion;
Chris@1372 816 }
Chris@1372 817
Chris@1372 818 blockFrame += stepSize;
Chris@1372 819
Chris@320 820 }
Chris@320 821
Chris@1372 822 if (!m_abandoned) {
Chris@1372 823 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@497 824
Chris@1372 825 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 826 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 827 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 828 addFeature(j, blockFrame, feature);
Chris@1372 829 }
Chris@850 830 }
Chris@850 831 }
Chris@1372 832 } catch (const std::exception &e) {
Chris@1372 833 SVCERR << "FeatureExtractionModelTransformer::run: Exception caught: "
Chris@1372 834 << e.what() << endl;
Chris@1372 835 m_abandoned = true;
Chris@1372 836 m_message = e.what();
Chris@497 837 }
Chris@320 838
Chris@850 839 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 840 setCompletion(j, 100);
Chris@850 841 }
Chris@320 842
Chris@320 843 if (frequencyDomain) {
Chris@930 844 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 845 delete fftModels[ch];
Chris@320 846 }
Chris@556 847 delete[] reals;
Chris@556 848 delete[] imaginaries;
Chris@320 849 }
Chris@974 850
Chris@974 851 for (int ch = 0; ch < channelCount; ++ch) {
Chris@974 852 delete[] buffers[ch];
Chris@974 853 }
Chris@974 854 delete[] buffers;
Chris@1237 855
Chris@1237 856 deinitialise();
Chris@320 857 }
Chris@320 858
Chris@320 859 void
Chris@363 860 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@1039 861 sv_frame_t startFrame,
Chris@1039 862 sv_frame_t size,
Chris@363 863 float **buffers)
Chris@320 864 {
Chris@1039 865 sv_frame_t offset = 0;
Chris@320 866
Chris@320 867 if (startFrame < 0) {
Chris@363 868 for (int c = 0; c < channelCount; ++c) {
Chris@1039 869 for (sv_frame_t i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 870 buffers[c][i] = 0.0f;
Chris@363 871 }
Chris@320 872 }
Chris@320 873 offset = -startFrame;
Chris@320 874 size -= offset;
Chris@320 875 if (size <= 0) return;
Chris@320 876 startFrame = 0;
Chris@320 877 }
Chris@320 878
Chris@1739 879 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
Chris@1739 880 if (!input) {
Chris@1739 881 return;
Chris@1739 882 }
Chris@363 883
Chris@1039 884 sv_frame_t got = 0;
Chris@350 885
Chris@363 886 if (channelCount == 1) {
Chris@363 887
Chris@1096 888 auto data = input->getData(m_input.getChannel(), startFrame, size);
Chris@1096 889 got = data.size();
Chris@1096 890
Chris@1096 891 copy(data.begin(), data.end(), buffers[0] + offset);
Chris@363 892
Chris@363 893 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 894 // use mean instead of sum, as plugin input
Chris@363 895 float cc = float(input->getChannelCount());
Chris@1096 896 for (sv_frame_t i = 0; i < got; ++i) {
Chris@363 897 buffers[0][i + offset] /= cc;
Chris@363 898 }
Chris@363 899 }
Chris@363 900
Chris@363 901 } else {
Chris@363 902
Chris@1096 903 auto data = input->getMultiChannelData(0, channelCount-1, startFrame, size);
Chris@1096 904 if (!data.empty()) {
Chris@1096 905 got = data[0].size();
Chris@1096 906 for (int c = 0; in_range_for(data, c); ++c) {
Chris@1096 907 copy(data[c].begin(), data[c].end(), buffers[c] + offset);
Chris@363 908 }
Chris@363 909 }
Chris@363 910 }
Chris@320 911
Chris@320 912 while (got < size) {
Chris@363 913 for (int c = 0; c < channelCount; ++c) {
Chris@363 914 buffers[c][got + offset] = 0.0;
Chris@363 915 }
Chris@320 916 ++got;
Chris@320 917 }
Chris@320 918 }
Chris@320 919
Chris@320 920 void
Chris@850 921 FeatureExtractionModelTransformer::addFeature(int n,
Chris@1039 922 sv_frame_t blockFrame,
Chris@850 923 const Vamp::Plugin::Feature &feature)
Chris@320 924 {
Chris@1739 925 auto input = ModelById::get(getInputModel());
Chris@1739 926 if (!input) return;
Chris@1739 927
Chris@1739 928 sv_samplerate_t inputRate = input->getSampleRate();
Chris@320 929
Chris@843 930 // cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
Chris@712 931 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
Chris@712 932 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
Chris@712 933 // << feature.hasDuration << ", duration = " << feature.duration
Chris@843 934 // << endl;
Chris@320 935
Chris@1039 936 sv_frame_t frame = blockFrame;
Chris@320 937
Chris@1739 938 if (m_descriptors[n].sampleType ==
Chris@1429 939 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 940
Chris@1429 941 if (!feature.hasTimestamp) {
Chris@1429 942 SVDEBUG
Chris@1429 943 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@1429 944 << "Feature has variable sample rate but no timestamp!"
Chris@1429 945 << endl;
Chris@1429 946 return;
Chris@1429 947 } else {
Chris@1429 948 frame = RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@1429 949 }
Chris@320 950
Chris@1071 951 // cerr << "variable sample rate: timestamp = " << feature.timestamp
Chris@1071 952 // << " at input rate " << inputRate << " -> " << frame << endl;
Chris@1071 953
Chris@1739 954 } else if (m_descriptors[n].sampleType ==
Chris@1429 955 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 956
Chris@1739 957 sv_samplerate_t rate = m_descriptors[n].sampleRate;
Chris@1071 958 if (rate <= 0.0) {
Chris@1071 959 rate = inputRate;
Chris@1071 960 }
Chris@1071 961
Chris@779 962 if (!feature.hasTimestamp) {
Chris@849 963 ++m_fixedRateFeatureNos[n];
Chris@779 964 } else {
Chris@779 965 RealTime ts(feature.timestamp.sec, feature.timestamp.nsec);
Chris@1071 966 m_fixedRateFeatureNos[n] = (int)lrint(ts.toDouble() * rate);
Chris@779 967 }
Chris@862 968
Chris@1071 969 // cerr << "m_fixedRateFeatureNo = " << m_fixedRateFeatureNos[n]
Chris@1739 970 // << ", m_descriptor->sampleRate = " << m_descriptors[n].sampleRate
Chris@862 971 // << ", inputRate = " << inputRate
Chris@862 972 // << " giving frame = ";
Chris@1071 973 frame = lrint((double(m_fixedRateFeatureNos[n]) / rate) * inputRate);
Chris@1071 974 // cerr << frame << endl;
Chris@320 975 }
Chris@862 976
Chris@862 977 if (frame < 0) {
Chris@1264 978 SVDEBUG
Chris@862 979 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@862 980 << "Negative frame counts are not supported (frame = " << frame
Chris@862 981 << " from timestamp " << feature.timestamp
Chris@862 982 << "), dropping feature"
Chris@862 983 << endl;
Chris@862 984 return;
Chris@862 985 }
Chris@862 986
Chris@441 987 // Rather than repeat the complicated tests from the constructor
Chris@441 988 // to determine what sort of model we must be adding the features
Chris@441 989 // to, we instead test what sort of model the constructor decided
Chris@441 990 // to create.
Chris@320 991
Chris@1739 992 ModelId outputId = m_outputs[n];
Chris@1739 993 bool found = false;
Chris@1739 994
Chris@1739 995 if (!found) {
Chris@1739 996 auto model = ModelById::getAs<SparseOneDimensionalModel>(outputId);
Chris@1739 997 if (model) {
Chris@1739 998 found = true;
Chris@1739 999 model->add(Event(frame, feature.label.c_str()));
Chris@1739 1000 }
Chris@1739 1001 }
Chris@1739 1002
Chris@1739 1003 if (!found) {
Chris@1739 1004 auto model = ModelById::getAs<SparseTimeValueModel>(outputId);
Chris@1739 1005 if (model) {
Chris@1739 1006 found = true;
Chris@441 1007
Chris@1739 1008 for (int i = 0; in_range_for(feature.values, i); ++i) {
Chris@350 1009
Chris@1739 1010 float value = feature.values[i];
Chris@320 1011
Chris@1739 1012 QString label = feature.label.c_str();
Chris@1739 1013 if (feature.values.size() > 1) {
Chris@1739 1014 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@1739 1015 }
Chris@350 1016
Chris@1739 1017 auto targetModel = model;
Chris@454 1018
Chris@1739 1019 if (m_needAdditionalModels[n] && i > 0) {
Chris@1739 1020 targetModel = ModelById::getAs<SparseTimeValueModel>
Chris@1739 1021 (getAdditionalModel(n, i));
Chris@1739 1022 if (!targetModel) targetModel = model;
Chris@1739 1023 }
Chris@454 1024
Chris@1739 1025 targetModel->add(Event(frame, value, label));
Chris@1739 1026 }
Chris@1739 1027 }
Chris@1739 1028 }
Chris@1739 1029
Chris@1739 1030 if (!found) {
Chris@1739 1031 if (ModelById::getAs<NoteModel>(outputId) ||
Chris@1739 1032 ModelById::getAs<RegionModel>(outputId)) {
Chris@1739 1033 found = true;
Chris@1739 1034
Chris@1739 1035 int index = 0;
Chris@1739 1036
Chris@1739 1037 float value = 0.0;
Chris@1739 1038 if ((int)feature.values.size() > index) {
Chris@1739 1039 value = feature.values[index++];
Chris@454 1040 }
Chris@454 1041
Chris@1739 1042 sv_frame_t duration = 1;
Chris@1739 1043 if (feature.hasDuration) {
Chris@1739 1044 duration = RealTime::realTime2Frame(feature.duration, inputRate);
Chris@1739 1045 } else {
Chris@1739 1046 if (in_range_for(feature.values, index)) {
Chris@1739 1047 duration = lrintf(feature.values[index++]);
Chris@1739 1048 }
Chris@876 1049 }
Chris@876 1050
Chris@1739 1051 auto noteModel = ModelById::getAs<NoteModel>(outputId);
Chris@1739 1052 if (noteModel) {
Chris@320 1053
Chris@1739 1054 float velocity = 100;
Chris@1739 1055 if ((int)feature.values.size() > index) {
Chris@1739 1056 velocity = feature.values[index++];
Chris@1739 1057 }
Chris@1739 1058 if (velocity < 0) velocity = 127;
Chris@1739 1059 if (velocity > 127) velocity = 127;
Chris@320 1060
Chris@1739 1061 noteModel->add(Event(frame, value, // value is pitch
Chris@1739 1062 duration,
Chris@1739 1063 velocity / 127.f,
Chris@1739 1064 feature.label.c_str()));
Chris@1739 1065 }
Chris@441 1066
Chris@1739 1067 auto regionModel = ModelById::getAs<RegionModel>(outputId);
Chris@1739 1068 if (regionModel) {
Chris@320 1069
Chris@1739 1070 if (feature.hasDuration && !feature.values.empty()) {
Chris@1739 1071
Chris@1739 1072 for (int i = 0; in_range_for(feature.values, i); ++i) {
Chris@1739 1073
Chris@1739 1074 float value = feature.values[i];
Chris@1739 1075
Chris@1739 1076 QString label = feature.label.c_str();
Chris@1739 1077 if (feature.values.size() > 1) {
Chris@1739 1078 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@1739 1079 }
Chris@1739 1080
Chris@1739 1081 regionModel->add(Event(frame,
Chris@1739 1082 value,
Chris@1739 1083 duration,
Chris@1739 1084 label));
Chris@1739 1085 }
Chris@1739 1086 } else {
Chris@1739 1087
Chris@1739 1088 regionModel->add(Event(frame,
Chris@1739 1089 value,
Chris@1739 1090 duration,
Chris@1739 1091 feature.label.c_str()));
Chris@1739 1092 }
Chris@441 1093 }
Chris@441 1094 }
Chris@1739 1095 }
gyorgyf@786 1096
Chris@1739 1097 if (!found) {
Chris@1739 1098 auto model = ModelById::getAs
Chris@1739 1099 <EditableDenseThreeDimensionalModel>(outputId);
Chris@1739 1100 if (model) {
Chris@1739 1101 found = true;
Chris@1739 1102
Chris@1739 1103 DenseThreeDimensionalModel::Column values = feature.values;
Chris@1739 1104
Chris@1739 1105 if (!feature.hasTimestamp && m_fixedRateFeatureNos[n] >= 0) {
Chris@1739 1106 model->setColumn(m_fixedRateFeatureNos[n], values);
Chris@454 1107 } else {
Chris@1739 1108 model->setColumn(int(frame / model->getResolution()), values);
Chris@454 1109 }
Chris@441 1110 }
Chris@1739 1111 }
Chris@320 1112
Chris@1739 1113 if (!found) {
Chris@690 1114 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
Chris@320 1115 }
Chris@320 1116 }
Chris@320 1117
Chris@320 1118 void
Chris@850 1119 FeatureExtractionModelTransformer::setCompletion(int n, int completion)
Chris@320 1120 {
Chris@1558 1121 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558 1122 SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
Chris@1558 1123 << completion << ")" << endl;
Chris@1558 1124 #endif
Chris@320 1125
Chris@1739 1126 ModelId outputId = m_outputs[n];
Chris@1739 1127 bool found = false;
Chris@1739 1128
Chris@1739 1129 if (!found) {
Chris@1739 1130 auto model = ModelById::getAs<SparseOneDimensionalModel>(outputId);
Chris@1739 1131 if (model) {
Chris@1739 1132 found = true;
Chris@1739 1133 model->setCompletion(completion, true);
Chris@1739 1134 }
Chris@1739 1135 }
Chris@320 1136
Chris@1739 1137 if (!found) {
Chris@1739 1138 auto model = ModelById::getAs<SparseTimeValueModel>(outputId);
Chris@1739 1139 if (model) {
Chris@1739 1140 found = true;
Chris@1739 1141 model->setCompletion(completion, true);
Chris@1739 1142 }
Chris@1739 1143 }
Chris@320 1144
Chris@1739 1145 if (!found) {
Chris@1739 1146 auto model = ModelById::getAs<NoteModel>(outputId);
Chris@1739 1147 if (model) {
Chris@1739 1148 found = true;
Chris@1739 1149 model->setCompletion(completion, true);
Chris@1739 1150 }
Chris@1739 1151 }
Chris@320 1152
Chris@1739 1153 if (!found) {
Chris@1739 1154 auto model = ModelById::getAs<RegionModel>(outputId);
Chris@1739 1155 if (model) {
Chris@1739 1156 found = true;
Chris@1739 1157 model->setCompletion(completion, true);
Chris@1739 1158 }
Chris@1739 1159 }
Chris@320 1160
Chris@1739 1161 if (!found) {
Chris@1739 1162 auto model = ModelById::getAs<EditableDenseThreeDimensionalModel>(outputId);
Chris@1739 1163 if (model) {
Chris@1739 1164 found = true;
Chris@1739 1165 model->setCompletion(completion, true);
Chris@1739 1166 }
Chris@320 1167 }
Chris@320 1168 }
Chris@320 1169