annotate transform/FeatureExtractionModelTransformer.cpp @ 1643:7a23dfe65d66 single-point

Update NoteModel to use EventSeries. This is incomplete and won't provide enough to update svgui for it yet; must also refactor to avoid duplication of nasty APIs when updating the rest of the models
author Chris Cannam
date Wed, 13 Mar 2019 14:50:10 +0000
parents 70e172e6cc59
children 513192aa9b03
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@1225 19
Chris@320 20 #include "plugin/PluginXml.h"
Chris@475 21 #include <vamp-hostsdk/Plugin.h>
Chris@320 22
Chris@320 23 #include "data/model/Model.h"
Chris@320 24 #include "base/Window.h"
Chris@387 25 #include "base/Exceptions.h"
Chris@320 26 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 27 #include "data/model/SparseTimeValueModel.h"
Chris@320 28 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 29 #include "data/model/DenseTimeValueModel.h"
Chris@320 30 #include "data/model/NoteModel.h"
gyorgyf@786 31 #include "data/model/FlexiNoteModel.h"
Chris@441 32 #include "data/model/RegionModel.h"
Chris@320 33 #include "data/model/FFTModel.h"
Chris@320 34 #include "data/model/WaveFileModel.h"
Chris@558 35 #include "rdf/PluginRDFDescription.h"
Chris@320 36
Chris@350 37 #include "TransformFactory.h"
Chris@350 38
Chris@320 39 #include <iostream>
Chris@320 40
Chris@859 41 #include <QSettings>
Chris@859 42
Chris@1558 43 //#define DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 1
Chris@1558 44
Chris@350 45 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 46 const Transform &transform) :
Chris@350 47 ModelTransformer(in, transform),
Chris@1582 48 m_plugin(nullptr),
Chris@1211 49 m_haveOutputs(false)
Chris@320 50 {
Chris@1080 51 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@849 52 }
Chris@849 53
Chris@849 54 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 55 const Transforms &transforms) :
Chris@849 56 ModelTransformer(in, transforms),
Chris@1582 57 m_plugin(nullptr),
Chris@1211 58 m_haveOutputs(false)
Chris@849 59 {
Chris@1080 60 if (m_transforms.empty()) {
Chris@1080 61 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s)" << endl;
Chris@1080 62 } else {
Chris@1080 63 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s), first has plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@1080 64 }
Chris@849 65 }
Chris@849 66
Chris@849 67 static bool
Chris@849 68 areTransformsSimilar(const Transform &t1, const Transform &t2)
Chris@849 69 {
Chris@849 70 Transform t2o(t2);
Chris@849 71 t2o.setOutput(t1.getOutput());
Chris@849 72 return t1 == t2o;
Chris@849 73 }
Chris@849 74
Chris@849 75 bool
Chris@849 76 FeatureExtractionModelTransformer::initialise()
Chris@849 77 {
Chris@1237 78 // This is (now) called from the run thread. The plugin is
Chris@1237 79 // constructed, initialised, used, and destroyed all from a single
Chris@1237 80 // thread.
Chris@1237 81
Chris@849 82 // All transforms must use the same plugin, parameters, and
Chris@849 83 // inputs: they can differ only in choice of plugin output. So we
Chris@849 84 // initialise based purely on the first transform in the list (but
Chris@849 85 // first check that they are actually similar as promised)
Chris@849 86
Chris@849 87 for (int j = 1; j < (int)m_transforms.size(); ++j) {
Chris@849 88 if (!areTransformsSimilar(m_transforms[0], m_transforms[j])) {
Chris@849 89 m_message = tr("Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output");
Chris@1368 90 SVCERR << m_message << endl;
Chris@849 91 return false;
Chris@849 92 }
Chris@849 93 }
Chris@849 94
Chris@849 95 Transform primaryTransform = m_transforms[0];
Chris@849 96
Chris@849 97 QString pluginId = primaryTransform.getPluginIdentifier();
Chris@320 98
Chris@1226 99 FeatureExtractionPluginFactory *factory =
Chris@1226 100 FeatureExtractionPluginFactory::instance();
Chris@320 101
Chris@320 102 if (!factory) {
Chris@361 103 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@1368 104 SVCERR << m_message << endl;
Chris@1429 105 return false;
Chris@320 106 }
Chris@320 107
Chris@350 108 DenseTimeValueModel *input = getConformingInput();
Chris@350 109 if (!input) {
Chris@361 110 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@1368 111 SVCERR << m_message << endl;
Chris@849 112 return false;
Chris@350 113 }
Chris@320 114
Chris@1264 115 SVDEBUG << "FeatureExtractionModelTransformer: Instantiating plugin for transform in thread "
Chris@1264 116 << QThread::currentThreadId() << endl;
Chris@1211 117
Chris@1040 118 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 119 if (!m_plugin) {
Chris@361 120 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@1368 121 SVCERR << m_message << endl;
Chris@1429 122 return false;
Chris@320 123 }
Chris@320 124
Chris@350 125 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@849 126 (primaryTransform, m_plugin);
Chris@1368 127
Chris@350 128 TransformFactory::getInstance()->setPluginParameters
Chris@849 129 (primaryTransform, m_plugin);
Chris@1368 130
Chris@930 131 int channelCount = input->getChannelCount();
Chris@930 132 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 133 channelCount = 1;
Chris@320 134 }
Chris@930 135 if ((int)m_plugin->getMinChannelCount() > channelCount) {
Chris@361 136 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 137 .arg(pluginId)
Chris@361 138 .arg(m_plugin->getMinChannelCount())
Chris@361 139 .arg(m_plugin->getMaxChannelCount())
Chris@361 140 .arg(input->getChannelCount());
Chris@1368 141 SVCERR << m_message << endl;
Chris@1429 142 return false;
Chris@320 143 }
Chris@1374 144
Chris@1374 145 int step = primaryTransform.getStepSize();
Chris@1374 146 int block = primaryTransform.getBlockSize();
Chris@1368 147
Chris@690 148 SVDEBUG << "Initialising feature extraction plugin with channels = "
Chris@1374 149 << channelCount << ", step = " << step
Chris@1374 150 << ", block = " << block << endl;
Chris@320 151
Chris@1374 152 if (!m_plugin->initialise(channelCount, step, block)) {
Chris@1374 153
Chris@1374 154 int preferredStep = int(m_plugin->getPreferredStepSize());
Chris@1374 155 int preferredBlock = int(m_plugin->getPreferredBlockSize());
Chris@1264 156
Chris@1374 157 if (step != preferredStep || block != preferredBlock) {
Chris@361 158
Chris@1374 159 SVDEBUG << "Initialisation failed, trying again with preferred step = "
Chris@1374 160 << preferredStep << ", block = " << preferredBlock << endl;
Chris@361 161
Chris@1374 162 if (!m_plugin->initialise(channelCount, preferredStep, preferredBlock)) {
Chris@361 163
Chris@1264 164 SVDEBUG << "Initialisation failed again" << endl;
Chris@1264 165
Chris@361 166 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 167 SVCERR << m_message << endl;
Chris@849 168 return false;
Chris@361 169
Chris@361 170 } else {
Chris@1264 171
Chris@1264 172 SVDEBUG << "Initialisation succeeded this time" << endl;
Chris@1374 173
Chris@1374 174 // Set these values into the primary transform in the list
Chris@1374 175 m_transforms[0].setStepSize(preferredStep);
Chris@1374 176 m_transforms[0].setBlockSize(preferredBlock);
Chris@1264 177
Chris@361 178 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 179 .arg(pluginId)
Chris@1374 180 .arg(step)
Chris@1374 181 .arg(block)
Chris@1374 182 .arg(preferredStep)
Chris@1374 183 .arg(preferredBlock);
Chris@1368 184 SVCERR << m_message << endl;
Chris@361 185 }
Chris@361 186
Chris@361 187 } else {
Chris@361 188
Chris@1374 189 SVDEBUG << "Initialisation failed (with step = " << step
Chris@1374 190 << " and block = " << block
Chris@1374 191 << ", both matching the plugin's preference)" << endl;
Chris@1264 192
Chris@361 193 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 194 SVCERR << m_message << endl;
Chris@849 195 return false;
Chris@361 196 }
Chris@1264 197 } else {
Chris@1264 198 SVDEBUG << "Initialisation succeeded" << endl;
Chris@320 199 }
Chris@320 200
Chris@849 201 if (primaryTransform.getPluginVersion() != "") {
Chris@366 202 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@849 203 if (pv != primaryTransform.getPluginVersion()) {
Chris@366 204 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@849 205 .arg(primaryTransform.getPluginVersion())
Chris@366 206 .arg(pluginId)
Chris@366 207 .arg(pv);
Chris@366 208 if (m_message != "") {
Chris@366 209 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 210 } else {
Chris@366 211 m_message = vm;
Chris@366 212 }
Chris@1368 213 SVCERR << m_message << endl;
Chris@366 214 }
Chris@366 215 }
Chris@366 216
Chris@320 217 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 218
Chris@320 219 if (outputs.empty()) {
Chris@361 220 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@1368 221 SVCERR << m_message << endl;
Chris@1429 222 return false;
Chris@320 223 }
Chris@320 224
Chris@849 225 for (int j = 0; j < (int)m_transforms.size(); ++j) {
Chris@849 226
Chris@849 227 for (int i = 0; i < (int)outputs.size(); ++i) {
Chris@849 228 // SVDEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl;
Chris@849 229 if (m_transforms[j].getOutput() == "" ||
Chris@849 230 outputs[i].identifier == m_transforms[j].getOutput().toStdString()) {
Chris@849 231 m_outputNos.push_back(i);
Chris@849 232 m_descriptors.push_back(new Vamp::Plugin::OutputDescriptor(outputs[i]));
Chris@849 233 m_fixedRateFeatureNos.push_back(-1); // we increment before use
Chris@849 234 break;
Chris@849 235 }
Chris@849 236 }
Chris@849 237
Chris@930 238 if ((int)m_descriptors.size() <= j) {
Chris@849 239 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@849 240 .arg(pluginId)
Chris@849 241 .arg(m_transforms[j].getOutput());
Chris@1368 242 SVCERR << m_message << endl;
Chris@849 243 return false;
Chris@849 244 }
Chris@320 245 }
Chris@320 246
Chris@849 247 for (int j = 0; j < (int)m_transforms.size(); ++j) {
Chris@876 248 createOutputModels(j);
Chris@849 249 }
Chris@849 250
Chris@1211 251 m_outputMutex.lock();
Chris@1211 252 m_haveOutputs = true;
Chris@1211 253 m_outputsCondition.wakeAll();
Chris@1211 254 m_outputMutex.unlock();
Chris@1211 255
Chris@849 256 return true;
Chris@558 257 }
Chris@558 258
Chris@558 259 void
Chris@1237 260 FeatureExtractionModelTransformer::deinitialise()
Chris@1237 261 {
Chris@1264 262 SVDEBUG << "FeatureExtractionModelTransformer: deleting plugin for transform in thread "
Chris@1264 263 << QThread::currentThreadId() << endl;
Chris@1372 264
Chris@1372 265 try {
Chris@1372 266 delete m_plugin;
Chris@1372 267 } catch (const std::exception &e) {
Chris@1372 268 // A destructor shouldn't throw an exception. But at one point
Chris@1372 269 // (now fixed) our plugin stub destructor could have
Chris@1372 270 // accidentally done so, so just in case:
Chris@1372 271 SVCERR << "FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl;
Chris@1372 272 m_message = e.what();
Chris@1372 273 }
Chris@1582 274 m_plugin = nullptr;
Chris@1372 275
Chris@1237 276 for (int j = 0; j < (int)m_descriptors.size(); ++j) {
Chris@1237 277 delete m_descriptors[j];
Chris@1237 278 }
Chris@1237 279 }
Chris@1237 280
Chris@1237 281 void
Chris@876 282 FeatureExtractionModelTransformer::createOutputModels(int n)
Chris@558 283 {
Chris@558 284 DenseTimeValueModel *input = getConformingInput();
Chris@712 285
Chris@849 286 PluginRDFDescription description(m_transforms[n].getPluginIdentifier());
Chris@849 287 QString outputId = m_transforms[n].getOutput();
Chris@558 288
Chris@320 289 int binCount = 1;
Chris@320 290 float minValue = 0.0, maxValue = 0.0;
Chris@320 291 bool haveExtents = false;
Chris@876 292 bool haveBinCount = m_descriptors[n]->hasFixedBinCount;
Chris@876 293
Chris@876 294 if (haveBinCount) {
Chris@1429 295 binCount = (int)m_descriptors[n]->binCount;
Chris@320 296 }
Chris@320 297
Chris@876 298 m_needAdditionalModels[n] = false;
Chris@876 299
Chris@843 300 // cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@1429 301 // << binCount << endl;
Chris@320 302
Chris@849 303 if (binCount > 0 && m_descriptors[n]->hasKnownExtents) {
Chris@1429 304 minValue = m_descriptors[n]->minValue;
Chris@1429 305 maxValue = m_descriptors[n]->maxValue;
Chris@320 306 haveExtents = true;
Chris@320 307 }
Chris@320 308
Chris@1040 309 sv_samplerate_t modelRate = input->getSampleRate();
Chris@1254 310 sv_samplerate_t outputRate = modelRate;
Chris@930 311 int modelResolution = 1;
Chris@712 312
Chris@849 313 if (m_descriptors[n]->sampleType !=
Chris@785 314 Vamp::Plugin::OutputDescriptor::OneSamplePerStep) {
Chris@1254 315
Chris@1254 316 outputRate = m_descriptors[n]->sampleRate;
Chris@1254 317
Chris@1254 318 //!!! SV doesn't actually support display of models that have
Chris@1254 319 //!!! different underlying rates together -- so we always set
Chris@1254 320 //!!! the model rate to be the input model's rate, and adjust
Chris@1254 321 //!!! the resolution appropriately. We can't properly display
Chris@1254 322 //!!! data with a higher resolution than the base model at all
Chris@1254 323 if (outputRate > input->getSampleRate()) {
Chris@1264 324 SVDEBUG << "WARNING: plugin reports output sample rate as "
Chris@1264 325 << outputRate
Chris@1264 326 << " (can't display features with finer resolution than the input rate of "
Chris@1264 327 << modelRate << ")" << endl;
Chris@1254 328 outputRate = modelRate;
Chris@785 329 }
Chris@785 330 }
Chris@785 331
Chris@849 332 switch (m_descriptors[n]->sampleType) {
Chris@320 333
Chris@320 334 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@1429 335 if (outputRate != 0.0) {
Chris@1429 336 modelResolution = int(round(modelRate / outputRate));
Chris@1429 337 }
Chris@1429 338 break;
Chris@320 339
Chris@320 340 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@1429 341 modelResolution = m_transforms[n].getStepSize();
Chris@1429 342 break;
Chris@320 343
Chris@320 344 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@1254 345 if (outputRate <= 0.0) {
Chris@1264 346 SVDEBUG << "WARNING: Fixed sample-rate plugin reports invalid sample rate " << m_descriptors[n]->sampleRate << "; defaulting to input rate of " << input->getSampleRate() << endl;
Chris@1071 347 modelResolution = 1;
Chris@451 348 } else {
Chris@1254 349 modelResolution = int(round(modelRate / outputRate));
Chris@1254 350 // cerr << "modelRate = " << modelRate << ", descriptor rate = " << outputRate << ", modelResolution = " << modelResolution << endl;
Chris@451 351 }
Chris@1429 352 break;
Chris@320 353 }
Chris@320 354
Chris@441 355 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 356
Chris@1582 357 Model *out = nullptr;
Chris@849 358
Chris@441 359 if (binCount == 0 &&
Chris@849 360 (preDurationPlugin || !m_descriptors[n]->hasDuration)) {
Chris@320 361
Chris@445 362 // Anything with no value and no duration is an instant
Chris@445 363
Chris@849 364 out = new SparseOneDimensionalModel(modelRate, modelResolution, false);
Chris@558 365 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 366 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 367
Chris@441 368 } else if ((preDurationPlugin && binCount > 1 &&
Chris@849 369 (m_descriptors[n]->sampleType ==
Chris@441 370 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@849 371 (!preDurationPlugin && m_descriptors[n]->hasDuration)) {
Chris@441 372
Chris@441 373 // For plugins using the old v1 API without explicit duration,
Chris@441 374 // we treat anything that has multiple bins (i.e. that has the
Chris@441 375 // potential to have value and duration) and a variable sample
Chris@441 376 // rate as a note model, taking its values as pitch, duration
Chris@441 377 // and velocity (if present) respectively. This is the same
Chris@441 378 // behaviour as always applied by SV to these plugins in the
Chris@441 379 // past.
Chris@441 380
Chris@441 381 // For plugins with the newer API, we treat anything with
Chris@441 382 // duration as either a note model with pitch and velocity, or
Chris@441 383 // a region model.
Chris@441 384
Chris@441 385 // How do we know whether it's an interval or note model?
Chris@441 386 // What's the essential difference? Is a note model any
Chris@441 387 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 388 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 389 // note model always have velocity? This is a good question
Chris@441 390 // to be addressed by accompanying RDF, but for the moment we
Chris@441 391 // will do the following...
Chris@441 392
Chris@441 393 bool isNoteModel = false;
Chris@441 394
Chris@441 395 // Regions have only value (and duration -- we can't extract a
Chris@441 396 // region model from an old-style plugin that doesn't support
Chris@441 397 // duration)
Chris@441 398 if (binCount > 1) isNoteModel = true;
Chris@441 399
Chris@595 400 // Regions do not have units of Hz or MIDI things (a sweeping
Chris@595 401 // assumption!)
Chris@849 402 if (m_descriptors[n]->unit == "Hz" ||
Chris@849 403 m_descriptors[n]->unit.find("MIDI") != std::string::npos ||
Chris@849 404 m_descriptors[n]->unit.find("midi") != std::string::npos) {
Chris@595 405 isNoteModel = true;
Chris@595 406 }
Chris@441 407
Chris@441 408 // If we had a "sparse 3D model", we would have the additional
Chris@441 409 // problem of determining whether to use that here (if bin
Chris@441 410 // count > 1). But we don't.
Chris@441 411
Chris@859 412 QSettings settings;
Chris@859 413 settings.beginGroup("Transformer");
Chris@859 414 bool flexi = settings.value("use-flexi-note-model", false).toBool();
Chris@859 415 settings.endGroup();
Chris@859 416
Chris@859 417 cerr << "flexi = " << flexi << endl;
Chris@859 418
Chris@859 419 if (isNoteModel && !flexi) {
Chris@441 420
Chris@441 421 NoteModel *model;
Chris@441 422 if (haveExtents) {
Chris@859 423 model = new NoteModel
Chris@859 424 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 425 } else {
Chris@859 426 model = new NoteModel
Chris@859 427 (modelRate, modelResolution, false);
gyorgyf@786 428 }
Chris@849 429 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 430 out = model;
gyorgyf@786 431
Chris@859 432 } else if (isNoteModel && flexi) {
gyorgyf@786 433
gyorgyf@786 434 FlexiNoteModel *model;
gyorgyf@786 435 if (haveExtents) {
Chris@859 436 model = new FlexiNoteModel
Chris@859 437 (modelRate, modelResolution, minValue, maxValue, false);
gyorgyf@786 438 } else {
Chris@859 439 model = new FlexiNoteModel
Chris@859 440 (modelRate, modelResolution, false);
Chris@441 441 }
Chris@849 442 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 443 out = model;
Chris@441 444
Chris@441 445 } else {
Chris@441 446
Chris@441 447 RegionModel *model;
Chris@441 448 if (haveExtents) {
Chris@441 449 model = new RegionModel
Chris@441 450 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 451 } else {
Chris@441 452 model = new RegionModel
Chris@441 453 (modelRate, modelResolution, false);
Chris@441 454 }
Chris@849 455 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 456 out = model;
Chris@441 457 }
Chris@441 458
Chris@558 459 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 460 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 461
Chris@876 462 } else if (binCount == 1 ||
Chris@849 463 (m_descriptors[n]->sampleType ==
Chris@441 464 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 465
Chris@441 466 // Anything that is not a 1D, note, or interval model and that
Chris@441 467 // has only one value per result must be a sparse time value
Chris@441 468 // model.
Chris@441 469
Chris@441 470 // Anything that is not a 1D, note, or interval model and that
Chris@876 471 // has a variable sample rate is treated as a set of sparse
Chris@876 472 // time value models, one per output bin, because we lack a
Chris@441 473 // sparse 3D model.
Chris@320 474
Chris@876 475 // Anything that is not a 1D, note, or interval model and that
Chris@876 476 // has a fixed sample rate but an unknown number of values per
Chris@876 477 // result is also treated as a set of sparse time value models.
Chris@876 478
Chris@876 479 // For sets of sparse time value models, we create a single
Chris@876 480 // model first as the "standard" output and then create models
Chris@876 481 // for bins 1+ in the additional model map (mapping the output
Chris@876 482 // descriptor to a list of models indexed by bin-1). But we
Chris@876 483 // don't create the additional models yet, as this case has to
Chris@876 484 // work even if the number of bins is unknown at this point --
Chris@877 485 // we create an additional model (copying its parameters from
Chris@877 486 // the default one) each time a new bin is encountered.
Chris@876 487
Chris@876 488 if (!haveBinCount || binCount > 1) {
Chris@876 489 m_needAdditionalModels[n] = true;
Chris@876 490 }
Chris@876 491
Chris@320 492 SparseTimeValueModel *model;
Chris@320 493 if (haveExtents) {
Chris@320 494 model = new SparseTimeValueModel
Chris@320 495 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 496 } else {
Chris@320 497 model = new SparseTimeValueModel
Chris@320 498 (modelRate, modelResolution, false);
Chris@320 499 }
Chris@558 500
Chris@558 501 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@849 502 model->setScaleUnits(outputs[m_outputNos[n]].unit.c_str());
Chris@320 503
Chris@849 504 out = model;
Chris@320 505
Chris@558 506 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 507 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 508
Chris@441 509 } else {
Chris@320 510
Chris@441 511 // Anything that is not a 1D, note, or interval model and that
Chris@441 512 // has a fixed sample rate and more than one value per result
Chris@441 513 // must be a dense 3D model.
Chris@320 514
Chris@320 515 EditableDenseThreeDimensionalModel *model =
Chris@320 516 new EditableDenseThreeDimensionalModel
Chris@535 517 (modelRate, modelResolution, binCount,
Chris@535 518 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 519 false);
Chris@320 520
Chris@1429 521 if (!m_descriptors[n]->binNames.empty()) {
Chris@1429 522 std::vector<QString> names;
Chris@1429 523 for (int i = 0; i < (int)m_descriptors[n]->binNames.size(); ++i) {
Chris@1429 524 names.push_back(m_descriptors[n]->binNames[i].c_str());
Chris@1429 525 }
Chris@1429 526 model->setBinNames(names);
Chris@1429 527 }
Chris@320 528
Chris@849 529 out = model;
Chris@558 530
Chris@558 531 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
Chris@849 532 out->setRDFTypeURI(outputSignalTypeURI);
Chris@320 533 }
Chris@333 534
Chris@849 535 if (out) {
Chris@849 536 out->setSourceModel(input);
Chris@849 537 m_outputs.push_back(out);
Chris@849 538 }
Chris@320 539 }
Chris@320 540
Chris@1211 541 void
Chris@1211 542 FeatureExtractionModelTransformer::awaitOutputModels()
Chris@1211 543 {
Chris@1211 544 m_outputMutex.lock();
Chris@1368 545 while (!m_haveOutputs && !m_abandoned) {
Chris@1368 546 m_outputsCondition.wait(&m_outputMutex, 500);
Chris@1211 547 }
Chris@1211 548 m_outputMutex.unlock();
Chris@1211 549 }
Chris@1211 550
Chris@331 551 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 552 {
Chris@1237 553 // Parent class dtor set the abandoned flag and waited for the run
Chris@1237 554 // thread to exit; the run thread owns the plugin, and should have
Chris@1237 555 // destroyed it before exiting (via a call to deinitialise)
Chris@320 556 }
Chris@320 557
Chris@876 558 FeatureExtractionModelTransformer::Models
Chris@876 559 FeatureExtractionModelTransformer::getAdditionalOutputModels()
Chris@876 560 {
Chris@876 561 Models mm;
Chris@876 562 for (AdditionalModelMap::iterator i = m_additionalModels.begin();
Chris@876 563 i != m_additionalModels.end(); ++i) {
Chris@876 564 for (std::map<int, SparseTimeValueModel *>::iterator j =
Chris@876 565 i->second.begin();
Chris@876 566 j != i->second.end(); ++j) {
Chris@876 567 SparseTimeValueModel *m = j->second;
Chris@876 568 if (m) mm.push_back(m);
Chris@876 569 }
Chris@876 570 }
Chris@876 571 return mm;
Chris@876 572 }
Chris@876 573
Chris@877 574 bool
Chris@877 575 FeatureExtractionModelTransformer::willHaveAdditionalOutputModels()
Chris@877 576 {
Chris@877 577 for (std::map<int, bool>::const_iterator i =
Chris@877 578 m_needAdditionalModels.begin();
Chris@877 579 i != m_needAdditionalModels.end(); ++i) {
Chris@877 580 if (i->second) return true;
Chris@877 581 }
Chris@877 582 return false;
Chris@877 583 }
Chris@877 584
Chris@876 585 SparseTimeValueModel *
Chris@876 586 FeatureExtractionModelTransformer::getAdditionalModel(int n, int binNo)
Chris@876 587 {
Chris@893 588 // std::cerr << "getAdditionalModel(" << n << ", " << binNo << ")" << std::endl;
Chris@876 589
Chris@876 590 if (binNo == 0) {
Chris@876 591 std::cerr << "Internal error: binNo == 0 in getAdditionalModel (should be using primary model)" << std::endl;
Chris@1582 592 return nullptr;
Chris@876 593 }
Chris@876 594
Chris@1582 595 if (!m_needAdditionalModels[n]) return nullptr;
Chris@1582 596 if (!isOutput<SparseTimeValueModel>(n)) return nullptr;
Chris@876 597 if (m_additionalModels[n][binNo]) return m_additionalModels[n][binNo];
Chris@876 598
Chris@876 599 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): creating" << std::endl;
Chris@876 600
Chris@876 601 SparseTimeValueModel *baseModel = getConformingOutput<SparseTimeValueModel>(n);
Chris@1582 602 if (!baseModel) return nullptr;
Chris@876 603
Chris@876 604 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): (from " << baseModel << ")" << std::endl;
Chris@876 605
Chris@876 606 SparseTimeValueModel *additional =
Chris@876 607 new SparseTimeValueModel(baseModel->getSampleRate(),
Chris@876 608 baseModel->getResolution(),
Chris@876 609 baseModel->getValueMinimum(),
Chris@876 610 baseModel->getValueMaximum(),
Chris@876 611 false);
Chris@876 612
Chris@876 613 additional->setScaleUnits(baseModel->getScaleUnits());
Chris@876 614 additional->setRDFTypeURI(baseModel->getRDFTypeURI());
Chris@876 615
Chris@876 616 m_additionalModels[n][binNo] = additional;
Chris@876 617 return additional;
Chris@876 618 }
Chris@876 619
Chris@320 620 DenseTimeValueModel *
Chris@350 621 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 622 {
Chris@690 623 // SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
Chris@408 624
Chris@320 625 DenseTimeValueModel *dtvm =
Chris@1429 626 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 627 if (!dtvm) {
Chris@1429 628 SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
Chris@320 629 }
Chris@320 630 return dtvm;
Chris@320 631 }
Chris@320 632
Chris@320 633 void
Chris@331 634 FeatureExtractionModelTransformer::run()
Chris@320 635 {
Chris@1373 636 try {
Chris@1373 637 if (!initialise()) {
Chris@1373 638 abandon();
Chris@1373 639 return;
Chris@1373 640 }
Chris@1373 641 } catch (const std::exception &e) {
Chris@1368 642 abandon();
Chris@1373 643 m_message = e.what();
Chris@1368 644 return;
Chris@1368 645 }
Chris@1211 646
Chris@350 647 DenseTimeValueModel *input = getConformingInput();
Chris@1368 648 if (!input) {
Chris@1368 649 abandon();
Chris@1368 650 return;
Chris@1368 651 }
Chris@320 652
Chris@1368 653 if (m_outputs.empty()) {
Chris@1368 654 abandon();
Chris@1368 655 return;
Chris@1368 656 }
Chris@320 657
Chris@850 658 Transform primaryTransform = m_transforms[0];
Chris@850 659
Chris@497 660 while (!input->isReady() && !m_abandoned) {
Chris@1558 661 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
Chris@497 662 usleep(500000);
Chris@320 663 }
Chris@1558 664 SVDEBUG << "FeatureExtractionModelTransformer::run: Waited, ready = "
Chris@1558 665 << input->isReady() << ", m_abandoned = " << m_abandoned << endl;
Chris@497 666 if (m_abandoned) return;
Chris@320 667
Chris@1040 668 sv_samplerate_t sampleRate = input->getSampleRate();
Chris@320 669
Chris@930 670 int channelCount = input->getChannelCount();
Chris@930 671 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 672 channelCount = 1;
Chris@320 673 }
Chris@320 674
Chris@320 675 float **buffers = new float*[channelCount];
Chris@930 676 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1429 677 buffers[ch] = new float[primaryTransform.getBlockSize() + 2];
Chris@320 678 }
Chris@320 679
Chris@930 680 int stepSize = primaryTransform.getStepSize();
Chris@930 681 int blockSize = primaryTransform.getBlockSize();
Chris@350 682
Chris@320 683 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 684 Vamp::Plugin::FrequencyDomain);
Chris@1558 685
Chris@320 686 std::vector<FFTModel *> fftModels;
Chris@320 687
Chris@320 688 if (frequencyDomain) {
Chris@930 689 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 690 FFTModel *model = new FFTModel
Chris@350 691 (getConformingInput(),
Chris@350 692 channelCount == 1 ? m_input.getChannel() : ch,
Chris@850 693 primaryTransform.getWindowType(),
Chris@350 694 blockSize,
Chris@350 695 stepSize,
Chris@1090 696 blockSize);
Chris@1080 697 if (!model->isOK() || model->getError() != "") {
Chris@1080 698 QString err = model->getError();
Chris@320 699 delete model;
Chris@850 700 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 701 setCompletion(j, 100);
Chris@850 702 }
Chris@387 703 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@1080 704 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer: error is: " + err);
Chris@320 705 }
Chris@320 706 fftModels.push_back(model);
Chris@1080 707 cerr << "created model for channel " << ch << endl;
Chris@320 708 }
Chris@320 709 }
Chris@320 710
Chris@1040 711 sv_frame_t startFrame = m_input.getModel()->getStartFrame();
Chris@1040 712 sv_frame_t endFrame = m_input.getModel()->getEndFrame();
Chris@320 713
Chris@850 714 RealTime contextStartRT = primaryTransform.getStartTime();
Chris@850 715 RealTime contextDurationRT = primaryTransform.getDuration();
Chris@350 716
Chris@1040 717 sv_frame_t contextStart =
Chris@350 718 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 719
Chris@1040 720 sv_frame_t contextDuration =
Chris@350 721 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 722
Chris@320 723 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 724 contextStart = startFrame;
Chris@320 725 }
Chris@320 726
Chris@320 727 if (contextDuration == 0) {
Chris@320 728 contextDuration = endFrame - contextStart;
Chris@320 729 }
Chris@320 730 if (contextStart + contextDuration > endFrame) {
Chris@320 731 contextDuration = endFrame - contextStart;
Chris@320 732 }
Chris@320 733
Chris@1039 734 sv_frame_t blockFrame = contextStart;
Chris@320 735
Chris@320 736 long prevCompletion = 0;
Chris@320 737
Chris@850 738 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 739 setCompletion(j, 0);
Chris@850 740 }
Chris@320 741
Chris@1582 742 float *reals = nullptr;
Chris@1582 743 float *imaginaries = nullptr;
Chris@556 744 if (frequencyDomain) {
Chris@556 745 reals = new float[blockSize/2 + 1];
Chris@556 746 imaginaries = new float[blockSize/2 + 1];
Chris@556 747 }
Chris@556 748
Chris@678 749 QString error = "";
Chris@678 750
Chris@1372 751 try {
Chris@1372 752 while (!m_abandoned) {
Chris@320 753
Chris@1372 754 if (frequencyDomain) {
Chris@1372 755 if (blockFrame - int(blockSize)/2 >
Chris@1372 756 contextStart + contextDuration) break;
Chris@1372 757 } else {
Chris@1372 758 if (blockFrame >=
Chris@1372 759 contextStart + contextDuration) break;
Chris@1372 760 }
Chris@320 761
Chris@1558 762 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558 763 SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@1558 764 << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@1558 765 << blockSize << endl;
Chris@1558 766 #endif
Chris@1558 767
Chris@1372 768 int completion = int
Chris@1372 769 ((((blockFrame - contextStart) / stepSize) * 99) /
Chris@1372 770 (contextDuration / stepSize + 1));
Chris@320 771
Chris@1372 772 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 773
Chris@1372 774 if (frequencyDomain) {
Chris@1372 775 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1372 776 int column = int((blockFrame - startFrame) / stepSize);
Chris@1372 777 if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) {
Chris@1372 778 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 779 buffers[ch][i*2] = reals[i];
Chris@1372 780 buffers[ch][i*2+1] = imaginaries[i];
Chris@1372 781 }
Chris@1372 782 } else {
Chris@1372 783 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 784 buffers[ch][i*2] = 0.f;
Chris@1372 785 buffers[ch][i*2+1] = 0.f;
Chris@1372 786 }
Chris@1558 787 }
Chris@1558 788
Chris@1372 789 error = fftModels[ch]->getError();
Chris@1372 790 if (error != "") {
Chris@1372 791 SVCERR << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << endl;
Chris@1372 792 m_abandoned = true;
Chris@1372 793 m_message = error;
Chris@1372 794 break;
Chris@1008 795 }
Chris@1372 796 }
Chris@1372 797 } else {
Chris@1372 798 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@1372 799 }
Chris@1372 800
Chris@1372 801 if (m_abandoned) break;
Chris@1372 802
Chris@1372 803 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@1558 804 (buffers,
Chris@1558 805 RealTime::frame2RealTime(blockFrame, sampleRate)
Chris@1558 806 .toVampRealTime());
Chris@1558 807
Chris@1372 808 if (m_abandoned) break;
Chris@1372 809
Chris@1372 810 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 811 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 812 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 813 addFeature(j, blockFrame, feature);
Chris@678 814 }
Chris@363 815 }
Chris@1372 816
Chris@1372 817 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@1372 818 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 819 setCompletion(j, completion);
Chris@1372 820 }
Chris@1372 821 prevCompletion = completion;
Chris@1372 822 }
Chris@1372 823
Chris@1372 824 blockFrame += stepSize;
Chris@1372 825
Chris@320 826 }
Chris@320 827
Chris@1372 828 if (!m_abandoned) {
Chris@1372 829 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@497 830
Chris@1372 831 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 832 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 833 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 834 addFeature(j, blockFrame, feature);
Chris@1372 835 }
Chris@850 836 }
Chris@850 837 }
Chris@1372 838 } catch (const std::exception &e) {
Chris@1372 839 SVCERR << "FeatureExtractionModelTransformer::run: Exception caught: "
Chris@1372 840 << e.what() << endl;
Chris@1372 841 m_abandoned = true;
Chris@1372 842 m_message = e.what();
Chris@497 843 }
Chris@320 844
Chris@850 845 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 846 setCompletion(j, 100);
Chris@850 847 }
Chris@320 848
Chris@320 849 if (frequencyDomain) {
Chris@930 850 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 851 delete fftModels[ch];
Chris@320 852 }
Chris@556 853 delete[] reals;
Chris@556 854 delete[] imaginaries;
Chris@320 855 }
Chris@974 856
Chris@974 857 for (int ch = 0; ch < channelCount; ++ch) {
Chris@974 858 delete[] buffers[ch];
Chris@974 859 }
Chris@974 860 delete[] buffers;
Chris@1237 861
Chris@1237 862 deinitialise();
Chris@320 863 }
Chris@320 864
Chris@320 865 void
Chris@363 866 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@1039 867 sv_frame_t startFrame,
Chris@1039 868 sv_frame_t size,
Chris@363 869 float **buffers)
Chris@320 870 {
Chris@1039 871 sv_frame_t offset = 0;
Chris@320 872
Chris@320 873 if (startFrame < 0) {
Chris@363 874 for (int c = 0; c < channelCount; ++c) {
Chris@1039 875 for (sv_frame_t i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 876 buffers[c][i] = 0.0f;
Chris@363 877 }
Chris@320 878 }
Chris@320 879 offset = -startFrame;
Chris@320 880 size -= offset;
Chris@320 881 if (size <= 0) return;
Chris@320 882 startFrame = 0;
Chris@320 883 }
Chris@320 884
Chris@350 885 DenseTimeValueModel *input = getConformingInput();
Chris@350 886 if (!input) return;
Chris@363 887
Chris@1039 888 sv_frame_t got = 0;
Chris@350 889
Chris@363 890 if (channelCount == 1) {
Chris@363 891
Chris@1096 892 auto data = input->getData(m_input.getChannel(), startFrame, size);
Chris@1096 893 got = data.size();
Chris@1096 894
Chris@1096 895 copy(data.begin(), data.end(), buffers[0] + offset);
Chris@363 896
Chris@363 897 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 898 // use mean instead of sum, as plugin input
Chris@363 899 float cc = float(input->getChannelCount());
Chris@1096 900 for (sv_frame_t i = 0; i < got; ++i) {
Chris@363 901 buffers[0][i + offset] /= cc;
Chris@363 902 }
Chris@363 903 }
Chris@363 904
Chris@363 905 } else {
Chris@363 906
Chris@1096 907 auto data = input->getMultiChannelData(0, channelCount-1, startFrame, size);
Chris@1096 908 if (!data.empty()) {
Chris@1096 909 got = data[0].size();
Chris@1096 910 for (int c = 0; in_range_for(data, c); ++c) {
Chris@1096 911 copy(data[c].begin(), data[c].end(), buffers[c] + offset);
Chris@363 912 }
Chris@363 913 }
Chris@363 914 }
Chris@320 915
Chris@320 916 while (got < size) {
Chris@363 917 for (int c = 0; c < channelCount; ++c) {
Chris@363 918 buffers[c][got + offset] = 0.0;
Chris@363 919 }
Chris@320 920 ++got;
Chris@320 921 }
Chris@320 922 }
Chris@320 923
Chris@320 924 void
Chris@850 925 FeatureExtractionModelTransformer::addFeature(int n,
Chris@1039 926 sv_frame_t blockFrame,
Chris@850 927 const Vamp::Plugin::Feature &feature)
Chris@320 928 {
Chris@1040 929 sv_samplerate_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 930
Chris@843 931 // cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
Chris@712 932 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
Chris@712 933 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
Chris@712 934 // << feature.hasDuration << ", duration = " << feature.duration
Chris@843 935 // << endl;
Chris@320 936
Chris@1039 937 sv_frame_t frame = blockFrame;
Chris@320 938
Chris@849 939 if (m_descriptors[n]->sampleType ==
Chris@1429 940 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 941
Chris@1429 942 if (!feature.hasTimestamp) {
Chris@1429 943 SVDEBUG
Chris@1429 944 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@1429 945 << "Feature has variable sample rate but no timestamp!"
Chris@1429 946 << endl;
Chris@1429 947 return;
Chris@1429 948 } else {
Chris@1429 949 frame = RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@1429 950 }
Chris@320 951
Chris@1071 952 // cerr << "variable sample rate: timestamp = " << feature.timestamp
Chris@1071 953 // << " at input rate " << inputRate << " -> " << frame << endl;
Chris@1071 954
Chris@849 955 } else if (m_descriptors[n]->sampleType ==
Chris@1429 956 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 957
Chris@1071 958 sv_samplerate_t rate = m_descriptors[n]->sampleRate;
Chris@1071 959 if (rate <= 0.0) {
Chris@1071 960 rate = inputRate;
Chris@1071 961 }
Chris@1071 962
Chris@779 963 if (!feature.hasTimestamp) {
Chris@849 964 ++m_fixedRateFeatureNos[n];
Chris@779 965 } else {
Chris@779 966 RealTime ts(feature.timestamp.sec, feature.timestamp.nsec);
Chris@1071 967 m_fixedRateFeatureNos[n] = (int)lrint(ts.toDouble() * rate);
Chris@779 968 }
Chris@862 969
Chris@1071 970 // cerr << "m_fixedRateFeatureNo = " << m_fixedRateFeatureNos[n]
Chris@1071 971 // << ", m_descriptor->sampleRate = " << m_descriptors[n]->sampleRate
Chris@862 972 // << ", inputRate = " << inputRate
Chris@862 973 // << " giving frame = ";
Chris@1071 974 frame = lrint((double(m_fixedRateFeatureNos[n]) / rate) * inputRate);
Chris@1071 975 // cerr << frame << endl;
Chris@320 976 }
Chris@862 977
Chris@862 978 if (frame < 0) {
Chris@1264 979 SVDEBUG
Chris@862 980 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@862 981 << "Negative frame counts are not supported (frame = " << frame
Chris@862 982 << " from timestamp " << feature.timestamp
Chris@862 983 << "), dropping feature"
Chris@862 984 << endl;
Chris@862 985 return;
Chris@862 986 }
Chris@862 987
Chris@441 988 // Rather than repeat the complicated tests from the constructor
Chris@441 989 // to determine what sort of model we must be adding the features
Chris@441 990 // to, we instead test what sort of model the constructor decided
Chris@441 991 // to create.
Chris@320 992
Chris@849 993 if (isOutput<SparseOneDimensionalModel>(n)) {
Chris@441 994
Chris@441 995 SparseOneDimensionalModel *model =
Chris@849 996 getConformingOutput<SparseOneDimensionalModel>(n);
Chris@1429 997 if (!model) return;
Chris@350 998
Chris@441 999 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 1000 (frame, feature.label.c_str()));
Chris@1429 1001
Chris@849 1002 } else if (isOutput<SparseTimeValueModel>(n)) {
Chris@320 1003
Chris@1429 1004 SparseTimeValueModel *model =
Chris@849 1005 getConformingOutput<SparseTimeValueModel>(n);
Chris@1429 1006 if (!model) return;
Chris@350 1007
Chris@930 1008 for (int i = 0; i < (int)feature.values.size(); ++i) {
Chris@454 1009
Chris@454 1010 float value = feature.values[i];
Chris@454 1011
Chris@454 1012 QString label = feature.label.c_str();
Chris@454 1013 if (feature.values.size() > 1) {
Chris@454 1014 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 1015 }
Chris@454 1016
Chris@876 1017 SparseTimeValueModel *targetModel = model;
Chris@876 1018
Chris@876 1019 if (m_needAdditionalModels[n] && i > 0) {
Chris@876 1020 targetModel = getAdditionalModel(n, i);
Chris@876 1021 if (!targetModel) targetModel = model;
Chris@893 1022 // std::cerr << "adding point to model " << targetModel
Chris@893 1023 // << " for output " << n << " bin " << i << std::endl;
Chris@876 1024 }
Chris@876 1025
Chris@876 1026 targetModel->addPoint
Chris@876 1027 (SparseTimeValueModel::Point(frame, value, label));
Chris@454 1028 }
Chris@320 1029
Chris@849 1030 } else if (isOutput<FlexiNoteModel>(n) || isOutput<NoteModel>(n) || isOutput<RegionModel>(n)) { //GF: Added Note Model
Chris@320 1031
Chris@441 1032 int index = 0;
Chris@441 1033
Chris@441 1034 float value = 0.0;
Chris@930 1035 if ((int)feature.values.size() > index) {
Chris@441 1036 value = feature.values[index++];
Chris@441 1037 }
Chris@320 1038
Chris@1039 1039 sv_frame_t duration = 1;
Chris@441 1040 if (feature.hasDuration) {
Chris@1040 1041 duration = RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 1042 } else {
Chris@1039 1043 if (in_range_for(feature.values, index)) {
Chris@1039 1044 duration = lrintf(feature.values[index++]);
Chris@441 1045 }
Chris@441 1046 }
gyorgyf@786 1047
Chris@891 1048 if (isOutput<FlexiNoteModel>(n)) { // GF: added for flexi note model
gyorgyf@786 1049
gyorgyf@786 1050 float velocity = 100;
Chris@930 1051 if ((int)feature.values.size() > index) {
gyorgyf@786 1052 velocity = feature.values[index++];
gyorgyf@786 1053 }
gyorgyf@786 1054 if (velocity < 0) velocity = 127;
gyorgyf@786 1055 if (velocity > 127) velocity = 127;
gyorgyf@786 1056
Chris@849 1057 FlexiNoteModel *model = getConformingOutput<FlexiNoteModel>(n);
gyorgyf@786 1058 if (!model) return;
Chris@1039 1059 model->addPoint(FlexiNoteModel::Point(frame,
Chris@1039 1060 value, // value is pitch
Chris@1039 1061 duration,
Chris@1039 1062 velocity / 127.f,
Chris@1039 1063 feature.label.c_str()));
Chris@1429 1064 // GF: end -- added for flexi note model
Chris@849 1065 } else if (isOutput<NoteModel>(n)) {
Chris@320 1066
Chris@441 1067 float velocity = 100;
Chris@930 1068 if ((int)feature.values.size() > index) {
Chris@441 1069 velocity = feature.values[index++];
Chris@441 1070 }
Chris@441 1071 if (velocity < 0) velocity = 127;
Chris@441 1072 if (velocity > 127) velocity = 127;
Chris@320 1073
Chris@849 1074 NoteModel *model = getConformingOutput<NoteModel>(n);
Chris@441 1075 if (!model) return;
Chris@1643 1076 model->addPoint(Event(frame, value, // value is pitch
Chris@1643 1077 duration,
Chris@1643 1078 velocity / 127.f,
Chris@1643 1079 feature.label.c_str()));
Chris@441 1080 } else {
gyorgyf@786 1081
Chris@849 1082 RegionModel *model = getConformingOutput<RegionModel>(n);
Chris@454 1083 if (!model) return;
Chris@454 1084
Chris@474 1085 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 1086
Chris@930 1087 for (int i = 0; i < (int)feature.values.size(); ++i) {
Chris@454 1088
Chris@454 1089 float value = feature.values[i];
Chris@454 1090
Chris@454 1091 QString label = feature.label.c_str();
Chris@454 1092 if (feature.values.size() > 1) {
Chris@454 1093 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 1094 }
Chris@454 1095
Chris@1039 1096 model->addPoint(RegionModel::Point(frame,
Chris@1039 1097 value,
Chris@1039 1098 duration,
Chris@454 1099 label));
Chris@454 1100 }
Chris@454 1101 } else {
Chris@454 1102
Chris@1039 1103 model->addPoint(RegionModel::Point(frame,
Chris@1039 1104 value,
Chris@1039 1105 duration,
Chris@441 1106 feature.label.c_str()));
Chris@454 1107 }
Chris@441 1108 }
Chris@1429 1109
Chris@849 1110 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) {
Chris@1429 1111
Chris@1429 1112 DenseThreeDimensionalModel::Column values = feature.values;
Chris@1429 1113
Chris@1429 1114 EditableDenseThreeDimensionalModel *model =
Chris@849 1115 getConformingOutput<EditableDenseThreeDimensionalModel>(n);
Chris@1429 1116 if (!model) return;
Chris@320 1117
Chris@889 1118 // cerr << "(note: model resolution = " << model->getResolution() << ")"
Chris@889 1119 // << endl;
Chris@889 1120
Chris@891 1121 if (!feature.hasTimestamp && m_fixedRateFeatureNos[n] >= 0) {
Chris@891 1122 model->setColumn(m_fixedRateFeatureNos[n], values);
Chris@889 1123 } else {
Chris@1039 1124 model->setColumn(int(frame / model->getResolution()), values);
Chris@889 1125 }
Chris@441 1126
Chris@441 1127 } else {
Chris@690 1128 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
Chris@320 1129 }
Chris@320 1130 }
Chris@320 1131
Chris@320 1132 void
Chris@850 1133 FeatureExtractionModelTransformer::setCompletion(int n, int completion)
Chris@320 1134 {
Chris@1558 1135 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
Chris@1558 1136 SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
Chris@1558 1137 << completion << ")" << endl;
Chris@1558 1138 #endif
Chris@320 1139
Chris@849 1140 if (isOutput<SparseOneDimensionalModel>(n)) {
Chris@320 1141
Chris@1429 1142 SparseOneDimensionalModel *model =
Chris@849 1143 getConformingOutput<SparseOneDimensionalModel>(n);
Chris@1429 1144 if (!model) return;
Chris@923 1145 if (model->isAbandoning()) abandon();
Chris@1429 1146 model->setCompletion(completion, true);
Chris@320 1147
Chris@849 1148 } else if (isOutput<SparseTimeValueModel>(n)) {
Chris@320 1149
Chris@1429 1150 SparseTimeValueModel *model =
Chris@849 1151 getConformingOutput<SparseTimeValueModel>(n);
Chris@1429 1152 if (!model) return;
Chris@923 1153 if (model->isAbandoning()) abandon();
Chris@1429 1154 model->setCompletion(completion, true);
Chris@320 1155
Chris@849 1156 } else if (isOutput<NoteModel>(n)) {
Chris@320 1157
Chris@1429 1158 NoteModel *model = getConformingOutput<NoteModel>(n);
Chris@1429 1159 if (!model) return;
Chris@923 1160 if (model->isAbandoning()) abandon();
Chris@1429 1161 model->setCompletion(completion, true);
Chris@1429 1162
Chris@923 1163 } else if (isOutput<FlexiNoteModel>(n)) {
gyorgyf@786 1164
Chris@1429 1165 FlexiNoteModel *model = getConformingOutput<FlexiNoteModel>(n);
Chris@1429 1166 if (!model) return;
Chris@923 1167 if (model->isAbandoning()) abandon();
Chris@1429 1168 model->setCompletion(completion, true);
Chris@320 1169
Chris@849 1170 } else if (isOutput<RegionModel>(n)) {
Chris@441 1171
Chris@1429 1172 RegionModel *model = getConformingOutput<RegionModel>(n);
Chris@1429 1173 if (!model) return;
Chris@923 1174 if (model->isAbandoning()) abandon();
Chris@1429 1175 model->setCompletion(completion, true);
Chris@441 1176
Chris@849 1177 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) {
Chris@320 1178
Chris@1429 1179 EditableDenseThreeDimensionalModel *model =
Chris@849 1180 getConformingOutput<EditableDenseThreeDimensionalModel>(n);
Chris@1429 1181 if (!model) return;
Chris@923 1182 if (model->isAbandoning()) abandon();
Chris@1429 1183 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 1184 }
Chris@320 1185 }
Chris@320 1186