annotate transform/FeatureExtractionModelTransformer.cpp @ 1429:48e9f538e6e9

Untabify
author Chris Cannam
date Thu, 01 Mar 2018 18:02:22 +0000
parents 8db84bd61eaa
children 73b3dd65e0b3
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@1225 19
Chris@320 20 #include "plugin/PluginXml.h"
Chris@475 21 #include <vamp-hostsdk/Plugin.h>
Chris@320 22
Chris@320 23 #include "data/model/Model.h"
Chris@320 24 #include "base/Window.h"
Chris@387 25 #include "base/Exceptions.h"
Chris@320 26 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 27 #include "data/model/SparseTimeValueModel.h"
Chris@320 28 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 29 #include "data/model/DenseTimeValueModel.h"
Chris@320 30 #include "data/model/NoteModel.h"
gyorgyf@786 31 #include "data/model/FlexiNoteModel.h"
Chris@441 32 #include "data/model/RegionModel.h"
Chris@320 33 #include "data/model/FFTModel.h"
Chris@320 34 #include "data/model/WaveFileModel.h"
Chris@558 35 #include "rdf/PluginRDFDescription.h"
Chris@320 36
Chris@350 37 #include "TransformFactory.h"
Chris@350 38
Chris@320 39 #include <iostream>
Chris@320 40
Chris@859 41 #include <QSettings>
Chris@859 42
Chris@350 43 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 44 const Transform &transform) :
Chris@350 45 ModelTransformer(in, transform),
Chris@1211 46 m_plugin(0),
Chris@1211 47 m_haveOutputs(false)
Chris@320 48 {
Chris@1080 49 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@849 50 }
Chris@849 51
Chris@849 52 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@859 53 const Transforms &transforms) :
Chris@849 54 ModelTransformer(in, transforms),
Chris@1211 55 m_plugin(0),
Chris@1211 56 m_haveOutputs(false)
Chris@849 57 {
Chris@1080 58 if (m_transforms.empty()) {
Chris@1080 59 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s)" << endl;
Chris@1080 60 } else {
Chris@1080 61 SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() << " transform(s), first has plugin " << m_transforms.begin()->getPluginIdentifier() << ", outputName " << m_transforms.begin()->getOutput() << endl;
Chris@1080 62 }
Chris@849 63 }
Chris@849 64
Chris@849 65 static bool
Chris@849 66 areTransformsSimilar(const Transform &t1, const Transform &t2)
Chris@849 67 {
Chris@849 68 Transform t2o(t2);
Chris@849 69 t2o.setOutput(t1.getOutput());
Chris@849 70 return t1 == t2o;
Chris@849 71 }
Chris@849 72
Chris@849 73 bool
Chris@849 74 FeatureExtractionModelTransformer::initialise()
Chris@849 75 {
Chris@1237 76 // This is (now) called from the run thread. The plugin is
Chris@1237 77 // constructed, initialised, used, and destroyed all from a single
Chris@1237 78 // thread.
Chris@1237 79
Chris@849 80 // All transforms must use the same plugin, parameters, and
Chris@849 81 // inputs: they can differ only in choice of plugin output. So we
Chris@849 82 // initialise based purely on the first transform in the list (but
Chris@849 83 // first check that they are actually similar as promised)
Chris@849 84
Chris@849 85 for (int j = 1; j < (int)m_transforms.size(); ++j) {
Chris@849 86 if (!areTransformsSimilar(m_transforms[0], m_transforms[j])) {
Chris@849 87 m_message = tr("Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output");
Chris@1368 88 SVCERR << m_message << endl;
Chris@849 89 return false;
Chris@849 90 }
Chris@849 91 }
Chris@849 92
Chris@849 93 Transform primaryTransform = m_transforms[0];
Chris@849 94
Chris@849 95 QString pluginId = primaryTransform.getPluginIdentifier();
Chris@320 96
Chris@1226 97 FeatureExtractionPluginFactory *factory =
Chris@1226 98 FeatureExtractionPluginFactory::instance();
Chris@320 99
Chris@320 100 if (!factory) {
Chris@361 101 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@1368 102 SVCERR << m_message << endl;
Chris@1429 103 return false;
Chris@320 104 }
Chris@320 105
Chris@350 106 DenseTimeValueModel *input = getConformingInput();
Chris@350 107 if (!input) {
Chris@361 108 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@1368 109 SVCERR << m_message << endl;
Chris@849 110 return false;
Chris@350 111 }
Chris@320 112
Chris@1264 113 SVDEBUG << "FeatureExtractionModelTransformer: Instantiating plugin for transform in thread "
Chris@1264 114 << QThread::currentThreadId() << endl;
Chris@1211 115
Chris@1040 116 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 117 if (!m_plugin) {
Chris@361 118 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@1368 119 SVCERR << m_message << endl;
Chris@1429 120 return false;
Chris@320 121 }
Chris@320 122
Chris@350 123 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@849 124 (primaryTransform, m_plugin);
Chris@1368 125
Chris@350 126 TransformFactory::getInstance()->setPluginParameters
Chris@849 127 (primaryTransform, m_plugin);
Chris@1368 128
Chris@930 129 int channelCount = input->getChannelCount();
Chris@930 130 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 131 channelCount = 1;
Chris@320 132 }
Chris@930 133 if ((int)m_plugin->getMinChannelCount() > channelCount) {
Chris@361 134 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 135 .arg(pluginId)
Chris@361 136 .arg(m_plugin->getMinChannelCount())
Chris@361 137 .arg(m_plugin->getMaxChannelCount())
Chris@361 138 .arg(input->getChannelCount());
Chris@1368 139 SVCERR << m_message << endl;
Chris@1429 140 return false;
Chris@320 141 }
Chris@1374 142
Chris@1374 143 int step = primaryTransform.getStepSize();
Chris@1374 144 int block = primaryTransform.getBlockSize();
Chris@1368 145
Chris@690 146 SVDEBUG << "Initialising feature extraction plugin with channels = "
Chris@1374 147 << channelCount << ", step = " << step
Chris@1374 148 << ", block = " << block << endl;
Chris@320 149
Chris@1374 150 if (!m_plugin->initialise(channelCount, step, block)) {
Chris@1374 151
Chris@1374 152 int preferredStep = int(m_plugin->getPreferredStepSize());
Chris@1374 153 int preferredBlock = int(m_plugin->getPreferredBlockSize());
Chris@1264 154
Chris@1374 155 if (step != preferredStep || block != preferredBlock) {
Chris@361 156
Chris@1374 157 SVDEBUG << "Initialisation failed, trying again with preferred step = "
Chris@1374 158 << preferredStep << ", block = " << preferredBlock << endl;
Chris@361 159
Chris@1374 160 if (!m_plugin->initialise(channelCount, preferredStep, preferredBlock)) {
Chris@361 161
Chris@1264 162 SVDEBUG << "Initialisation failed again" << endl;
Chris@1264 163
Chris@361 164 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 165 SVCERR << m_message << endl;
Chris@849 166 return false;
Chris@361 167
Chris@361 168 } else {
Chris@1264 169
Chris@1264 170 SVDEBUG << "Initialisation succeeded this time" << endl;
Chris@1374 171
Chris@1374 172 // Set these values into the primary transform in the list
Chris@1374 173 m_transforms[0].setStepSize(preferredStep);
Chris@1374 174 m_transforms[0].setBlockSize(preferredBlock);
Chris@1264 175
Chris@361 176 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 177 .arg(pluginId)
Chris@1374 178 .arg(step)
Chris@1374 179 .arg(block)
Chris@1374 180 .arg(preferredStep)
Chris@1374 181 .arg(preferredBlock);
Chris@1368 182 SVCERR << m_message << endl;
Chris@361 183 }
Chris@361 184
Chris@361 185 } else {
Chris@361 186
Chris@1374 187 SVDEBUG << "Initialisation failed (with step = " << step
Chris@1374 188 << " and block = " << block
Chris@1374 189 << ", both matching the plugin's preference)" << endl;
Chris@1264 190
Chris@361 191 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@1368 192 SVCERR << m_message << endl;
Chris@849 193 return false;
Chris@361 194 }
Chris@1264 195 } else {
Chris@1264 196 SVDEBUG << "Initialisation succeeded" << endl;
Chris@320 197 }
Chris@320 198
Chris@849 199 if (primaryTransform.getPluginVersion() != "") {
Chris@366 200 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@849 201 if (pv != primaryTransform.getPluginVersion()) {
Chris@366 202 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@849 203 .arg(primaryTransform.getPluginVersion())
Chris@366 204 .arg(pluginId)
Chris@366 205 .arg(pv);
Chris@366 206 if (m_message != "") {
Chris@366 207 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 208 } else {
Chris@366 209 m_message = vm;
Chris@366 210 }
Chris@1368 211 SVCERR << m_message << endl;
Chris@366 212 }
Chris@366 213 }
Chris@366 214
Chris@320 215 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 216
Chris@320 217 if (outputs.empty()) {
Chris@361 218 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@1368 219 SVCERR << m_message << endl;
Chris@1429 220 return false;
Chris@320 221 }
Chris@320 222
Chris@849 223 for (int j = 0; j < (int)m_transforms.size(); ++j) {
Chris@849 224
Chris@849 225 for (int i = 0; i < (int)outputs.size(); ++i) {
Chris@849 226 // SVDEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl;
Chris@849 227 if (m_transforms[j].getOutput() == "" ||
Chris@849 228 outputs[i].identifier == m_transforms[j].getOutput().toStdString()) {
Chris@849 229 m_outputNos.push_back(i);
Chris@849 230 m_descriptors.push_back(new Vamp::Plugin::OutputDescriptor(outputs[i]));
Chris@849 231 m_fixedRateFeatureNos.push_back(-1); // we increment before use
Chris@849 232 break;
Chris@849 233 }
Chris@849 234 }
Chris@849 235
Chris@930 236 if ((int)m_descriptors.size() <= j) {
Chris@849 237 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@849 238 .arg(pluginId)
Chris@849 239 .arg(m_transforms[j].getOutput());
Chris@1368 240 SVCERR << m_message << endl;
Chris@849 241 return false;
Chris@849 242 }
Chris@320 243 }
Chris@320 244
Chris@849 245 for (int j = 0; j < (int)m_transforms.size(); ++j) {
Chris@876 246 createOutputModels(j);
Chris@849 247 }
Chris@849 248
Chris@1211 249 m_outputMutex.lock();
Chris@1211 250 m_haveOutputs = true;
Chris@1211 251 m_outputsCondition.wakeAll();
Chris@1211 252 m_outputMutex.unlock();
Chris@1211 253
Chris@849 254 return true;
Chris@558 255 }
Chris@558 256
Chris@558 257 void
Chris@1237 258 FeatureExtractionModelTransformer::deinitialise()
Chris@1237 259 {
Chris@1264 260 SVDEBUG << "FeatureExtractionModelTransformer: deleting plugin for transform in thread "
Chris@1264 261 << QThread::currentThreadId() << endl;
Chris@1372 262
Chris@1372 263 try {
Chris@1372 264 delete m_plugin;
Chris@1372 265 } catch (const std::exception &e) {
Chris@1372 266 // A destructor shouldn't throw an exception. But at one point
Chris@1372 267 // (now fixed) our plugin stub destructor could have
Chris@1372 268 // accidentally done so, so just in case:
Chris@1372 269 SVCERR << "FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl;
Chris@1372 270 m_message = e.what();
Chris@1372 271 }
Chris@1372 272 m_plugin = 0;
Chris@1372 273
Chris@1237 274 for (int j = 0; j < (int)m_descriptors.size(); ++j) {
Chris@1237 275 delete m_descriptors[j];
Chris@1237 276 }
Chris@1237 277 }
Chris@1237 278
Chris@1237 279 void
Chris@876 280 FeatureExtractionModelTransformer::createOutputModels(int n)
Chris@558 281 {
Chris@558 282 DenseTimeValueModel *input = getConformingInput();
Chris@712 283
Chris@849 284 PluginRDFDescription description(m_transforms[n].getPluginIdentifier());
Chris@849 285 QString outputId = m_transforms[n].getOutput();
Chris@558 286
Chris@320 287 int binCount = 1;
Chris@320 288 float minValue = 0.0, maxValue = 0.0;
Chris@320 289 bool haveExtents = false;
Chris@876 290 bool haveBinCount = m_descriptors[n]->hasFixedBinCount;
Chris@876 291
Chris@876 292 if (haveBinCount) {
Chris@1429 293 binCount = (int)m_descriptors[n]->binCount;
Chris@320 294 }
Chris@320 295
Chris@876 296 m_needAdditionalModels[n] = false;
Chris@876 297
Chris@843 298 // cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@1429 299 // << binCount << endl;
Chris@320 300
Chris@849 301 if (binCount > 0 && m_descriptors[n]->hasKnownExtents) {
Chris@1429 302 minValue = m_descriptors[n]->minValue;
Chris@1429 303 maxValue = m_descriptors[n]->maxValue;
Chris@320 304 haveExtents = true;
Chris@320 305 }
Chris@320 306
Chris@1040 307 sv_samplerate_t modelRate = input->getSampleRate();
Chris@1254 308 sv_samplerate_t outputRate = modelRate;
Chris@930 309 int modelResolution = 1;
Chris@712 310
Chris@849 311 if (m_descriptors[n]->sampleType !=
Chris@785 312 Vamp::Plugin::OutputDescriptor::OneSamplePerStep) {
Chris@1254 313
Chris@1254 314 outputRate = m_descriptors[n]->sampleRate;
Chris@1254 315
Chris@1254 316 //!!! SV doesn't actually support display of models that have
Chris@1254 317 //!!! different underlying rates together -- so we always set
Chris@1254 318 //!!! the model rate to be the input model's rate, and adjust
Chris@1254 319 //!!! the resolution appropriately. We can't properly display
Chris@1254 320 //!!! data with a higher resolution than the base model at all
Chris@1254 321 if (outputRate > input->getSampleRate()) {
Chris@1264 322 SVDEBUG << "WARNING: plugin reports output sample rate as "
Chris@1264 323 << outputRate
Chris@1264 324 << " (can't display features with finer resolution than the input rate of "
Chris@1264 325 << modelRate << ")" << endl;
Chris@1254 326 outputRate = modelRate;
Chris@785 327 }
Chris@785 328 }
Chris@785 329
Chris@849 330 switch (m_descriptors[n]->sampleType) {
Chris@320 331
Chris@320 332 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@1429 333 if (outputRate != 0.0) {
Chris@1429 334 modelResolution = int(round(modelRate / outputRate));
Chris@1429 335 }
Chris@1429 336 break;
Chris@320 337
Chris@320 338 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@1429 339 modelResolution = m_transforms[n].getStepSize();
Chris@1429 340 break;
Chris@320 341
Chris@320 342 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@1254 343 if (outputRate <= 0.0) {
Chris@1264 344 SVDEBUG << "WARNING: Fixed sample-rate plugin reports invalid sample rate " << m_descriptors[n]->sampleRate << "; defaulting to input rate of " << input->getSampleRate() << endl;
Chris@1071 345 modelResolution = 1;
Chris@451 346 } else {
Chris@1254 347 modelResolution = int(round(modelRate / outputRate));
Chris@1254 348 // cerr << "modelRate = " << modelRate << ", descriptor rate = " << outputRate << ", modelResolution = " << modelResolution << endl;
Chris@451 349 }
Chris@1429 350 break;
Chris@320 351 }
Chris@320 352
Chris@441 353 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 354
Chris@849 355 Model *out = 0;
Chris@849 356
Chris@441 357 if (binCount == 0 &&
Chris@849 358 (preDurationPlugin || !m_descriptors[n]->hasDuration)) {
Chris@320 359
Chris@445 360 // Anything with no value and no duration is an instant
Chris@445 361
Chris@849 362 out = new SparseOneDimensionalModel(modelRate, modelResolution, false);
Chris@558 363 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 364 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 365
Chris@441 366 } else if ((preDurationPlugin && binCount > 1 &&
Chris@849 367 (m_descriptors[n]->sampleType ==
Chris@441 368 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@849 369 (!preDurationPlugin && m_descriptors[n]->hasDuration)) {
Chris@441 370
Chris@441 371 // For plugins using the old v1 API without explicit duration,
Chris@441 372 // we treat anything that has multiple bins (i.e. that has the
Chris@441 373 // potential to have value and duration) and a variable sample
Chris@441 374 // rate as a note model, taking its values as pitch, duration
Chris@441 375 // and velocity (if present) respectively. This is the same
Chris@441 376 // behaviour as always applied by SV to these plugins in the
Chris@441 377 // past.
Chris@441 378
Chris@441 379 // For plugins with the newer API, we treat anything with
Chris@441 380 // duration as either a note model with pitch and velocity, or
Chris@441 381 // a region model.
Chris@441 382
Chris@441 383 // How do we know whether it's an interval or note model?
Chris@441 384 // What's the essential difference? Is a note model any
Chris@441 385 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 386 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 387 // note model always have velocity? This is a good question
Chris@441 388 // to be addressed by accompanying RDF, but for the moment we
Chris@441 389 // will do the following...
Chris@441 390
Chris@441 391 bool isNoteModel = false;
Chris@441 392
Chris@441 393 // Regions have only value (and duration -- we can't extract a
Chris@441 394 // region model from an old-style plugin that doesn't support
Chris@441 395 // duration)
Chris@441 396 if (binCount > 1) isNoteModel = true;
Chris@441 397
Chris@595 398 // Regions do not have units of Hz or MIDI things (a sweeping
Chris@595 399 // assumption!)
Chris@849 400 if (m_descriptors[n]->unit == "Hz" ||
Chris@849 401 m_descriptors[n]->unit.find("MIDI") != std::string::npos ||
Chris@849 402 m_descriptors[n]->unit.find("midi") != std::string::npos) {
Chris@595 403 isNoteModel = true;
Chris@595 404 }
Chris@441 405
Chris@441 406 // If we had a "sparse 3D model", we would have the additional
Chris@441 407 // problem of determining whether to use that here (if bin
Chris@441 408 // count > 1). But we don't.
Chris@441 409
Chris@859 410 QSettings settings;
Chris@859 411 settings.beginGroup("Transformer");
Chris@859 412 bool flexi = settings.value("use-flexi-note-model", false).toBool();
Chris@859 413 settings.endGroup();
Chris@859 414
Chris@859 415 cerr << "flexi = " << flexi << endl;
Chris@859 416
Chris@859 417 if (isNoteModel && !flexi) {
Chris@441 418
Chris@441 419 NoteModel *model;
Chris@441 420 if (haveExtents) {
Chris@859 421 model = new NoteModel
Chris@859 422 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 423 } else {
Chris@859 424 model = new NoteModel
Chris@859 425 (modelRate, modelResolution, false);
gyorgyf@786 426 }
Chris@849 427 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 428 out = model;
gyorgyf@786 429
Chris@859 430 } else if (isNoteModel && flexi) {
gyorgyf@786 431
gyorgyf@786 432 FlexiNoteModel *model;
gyorgyf@786 433 if (haveExtents) {
Chris@859 434 model = new FlexiNoteModel
Chris@859 435 (modelRate, modelResolution, minValue, maxValue, false);
gyorgyf@786 436 } else {
Chris@859 437 model = new FlexiNoteModel
Chris@859 438 (modelRate, modelResolution, false);
Chris@441 439 }
Chris@849 440 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 441 out = model;
Chris@441 442
Chris@441 443 } else {
Chris@441 444
Chris@441 445 RegionModel *model;
Chris@441 446 if (haveExtents) {
Chris@441 447 model = new RegionModel
Chris@441 448 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 449 } else {
Chris@441 450 model = new RegionModel
Chris@441 451 (modelRate, modelResolution, false);
Chris@441 452 }
Chris@849 453 model->setScaleUnits(m_descriptors[n]->unit.c_str());
Chris@849 454 out = model;
Chris@441 455 }
Chris@441 456
Chris@558 457 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 458 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 459
Chris@876 460 } else if (binCount == 1 ||
Chris@849 461 (m_descriptors[n]->sampleType ==
Chris@441 462 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 463
Chris@441 464 // Anything that is not a 1D, note, or interval model and that
Chris@441 465 // has only one value per result must be a sparse time value
Chris@441 466 // model.
Chris@441 467
Chris@441 468 // Anything that is not a 1D, note, or interval model and that
Chris@876 469 // has a variable sample rate is treated as a set of sparse
Chris@876 470 // time value models, one per output bin, because we lack a
Chris@441 471 // sparse 3D model.
Chris@320 472
Chris@876 473 // Anything that is not a 1D, note, or interval model and that
Chris@876 474 // has a fixed sample rate but an unknown number of values per
Chris@876 475 // result is also treated as a set of sparse time value models.
Chris@876 476
Chris@876 477 // For sets of sparse time value models, we create a single
Chris@876 478 // model first as the "standard" output and then create models
Chris@876 479 // for bins 1+ in the additional model map (mapping the output
Chris@876 480 // descriptor to a list of models indexed by bin-1). But we
Chris@876 481 // don't create the additional models yet, as this case has to
Chris@876 482 // work even if the number of bins is unknown at this point --
Chris@877 483 // we create an additional model (copying its parameters from
Chris@877 484 // the default one) each time a new bin is encountered.
Chris@876 485
Chris@876 486 if (!haveBinCount || binCount > 1) {
Chris@876 487 m_needAdditionalModels[n] = true;
Chris@876 488 }
Chris@876 489
Chris@320 490 SparseTimeValueModel *model;
Chris@320 491 if (haveExtents) {
Chris@320 492 model = new SparseTimeValueModel
Chris@320 493 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 494 } else {
Chris@320 495 model = new SparseTimeValueModel
Chris@320 496 (modelRate, modelResolution, false);
Chris@320 497 }
Chris@558 498
Chris@558 499 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@849 500 model->setScaleUnits(outputs[m_outputNos[n]].unit.c_str());
Chris@320 501
Chris@849 502 out = model;
Chris@320 503
Chris@558 504 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@849 505 out->setRDFTypeURI(outputEventTypeURI);
Chris@558 506
Chris@441 507 } else {
Chris@320 508
Chris@441 509 // Anything that is not a 1D, note, or interval model and that
Chris@441 510 // has a fixed sample rate and more than one value per result
Chris@441 511 // must be a dense 3D model.
Chris@320 512
Chris@320 513 EditableDenseThreeDimensionalModel *model =
Chris@320 514 new EditableDenseThreeDimensionalModel
Chris@535 515 (modelRate, modelResolution, binCount,
Chris@535 516 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 517 false);
Chris@320 518
Chris@1429 519 if (!m_descriptors[n]->binNames.empty()) {
Chris@1429 520 std::vector<QString> names;
Chris@1429 521 for (int i = 0; i < (int)m_descriptors[n]->binNames.size(); ++i) {
Chris@1429 522 names.push_back(m_descriptors[n]->binNames[i].c_str());
Chris@1429 523 }
Chris@1429 524 model->setBinNames(names);
Chris@1429 525 }
Chris@320 526
Chris@849 527 out = model;
Chris@558 528
Chris@558 529 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
Chris@849 530 out->setRDFTypeURI(outputSignalTypeURI);
Chris@320 531 }
Chris@333 532
Chris@849 533 if (out) {
Chris@849 534 out->setSourceModel(input);
Chris@849 535 m_outputs.push_back(out);
Chris@849 536 }
Chris@320 537 }
Chris@320 538
Chris@1211 539 void
Chris@1211 540 FeatureExtractionModelTransformer::awaitOutputModels()
Chris@1211 541 {
Chris@1211 542 m_outputMutex.lock();
Chris@1368 543 while (!m_haveOutputs && !m_abandoned) {
Chris@1368 544 m_outputsCondition.wait(&m_outputMutex, 500);
Chris@1211 545 }
Chris@1211 546 m_outputMutex.unlock();
Chris@1211 547 }
Chris@1211 548
Chris@331 549 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 550 {
Chris@1237 551 // Parent class dtor set the abandoned flag and waited for the run
Chris@1237 552 // thread to exit; the run thread owns the plugin, and should have
Chris@1237 553 // destroyed it before exiting (via a call to deinitialise)
Chris@320 554 }
Chris@320 555
Chris@876 556 FeatureExtractionModelTransformer::Models
Chris@876 557 FeatureExtractionModelTransformer::getAdditionalOutputModels()
Chris@876 558 {
Chris@876 559 Models mm;
Chris@876 560 for (AdditionalModelMap::iterator i = m_additionalModels.begin();
Chris@876 561 i != m_additionalModels.end(); ++i) {
Chris@876 562 for (std::map<int, SparseTimeValueModel *>::iterator j =
Chris@876 563 i->second.begin();
Chris@876 564 j != i->second.end(); ++j) {
Chris@876 565 SparseTimeValueModel *m = j->second;
Chris@876 566 if (m) mm.push_back(m);
Chris@876 567 }
Chris@876 568 }
Chris@876 569 return mm;
Chris@876 570 }
Chris@876 571
Chris@877 572 bool
Chris@877 573 FeatureExtractionModelTransformer::willHaveAdditionalOutputModels()
Chris@877 574 {
Chris@877 575 for (std::map<int, bool>::const_iterator i =
Chris@877 576 m_needAdditionalModels.begin();
Chris@877 577 i != m_needAdditionalModels.end(); ++i) {
Chris@877 578 if (i->second) return true;
Chris@877 579 }
Chris@877 580 return false;
Chris@877 581 }
Chris@877 582
Chris@876 583 SparseTimeValueModel *
Chris@876 584 FeatureExtractionModelTransformer::getAdditionalModel(int n, int binNo)
Chris@876 585 {
Chris@893 586 // std::cerr << "getAdditionalModel(" << n << ", " << binNo << ")" << std::endl;
Chris@876 587
Chris@876 588 if (binNo == 0) {
Chris@876 589 std::cerr << "Internal error: binNo == 0 in getAdditionalModel (should be using primary model)" << std::endl;
Chris@876 590 return 0;
Chris@876 591 }
Chris@876 592
Chris@876 593 if (!m_needAdditionalModels[n]) return 0;
Chris@876 594 if (!isOutput<SparseTimeValueModel>(n)) return 0;
Chris@876 595 if (m_additionalModels[n][binNo]) return m_additionalModels[n][binNo];
Chris@876 596
Chris@876 597 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): creating" << std::endl;
Chris@876 598
Chris@876 599 SparseTimeValueModel *baseModel = getConformingOutput<SparseTimeValueModel>(n);
Chris@876 600 if (!baseModel) return 0;
Chris@876 601
Chris@876 602 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): (from " << baseModel << ")" << std::endl;
Chris@876 603
Chris@876 604 SparseTimeValueModel *additional =
Chris@876 605 new SparseTimeValueModel(baseModel->getSampleRate(),
Chris@876 606 baseModel->getResolution(),
Chris@876 607 baseModel->getValueMinimum(),
Chris@876 608 baseModel->getValueMaximum(),
Chris@876 609 false);
Chris@876 610
Chris@876 611 additional->setScaleUnits(baseModel->getScaleUnits());
Chris@876 612 additional->setRDFTypeURI(baseModel->getRDFTypeURI());
Chris@876 613
Chris@876 614 m_additionalModels[n][binNo] = additional;
Chris@876 615 return additional;
Chris@876 616 }
Chris@876 617
Chris@320 618 DenseTimeValueModel *
Chris@350 619 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 620 {
Chris@690 621 // SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
Chris@408 622
Chris@320 623 DenseTimeValueModel *dtvm =
Chris@1429 624 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 625 if (!dtvm) {
Chris@1429 626 SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
Chris@320 627 }
Chris@320 628 return dtvm;
Chris@320 629 }
Chris@320 630
Chris@320 631 void
Chris@331 632 FeatureExtractionModelTransformer::run()
Chris@320 633 {
Chris@1373 634 try {
Chris@1373 635 if (!initialise()) {
Chris@1373 636 abandon();
Chris@1373 637 return;
Chris@1373 638 }
Chris@1373 639 } catch (const std::exception &e) {
Chris@1368 640 abandon();
Chris@1373 641 m_message = e.what();
Chris@1368 642 return;
Chris@1368 643 }
Chris@1211 644
Chris@350 645 DenseTimeValueModel *input = getConformingInput();
Chris@1368 646 if (!input) {
Chris@1368 647 abandon();
Chris@1368 648 return;
Chris@1368 649 }
Chris@320 650
Chris@1368 651 if (m_outputs.empty()) {
Chris@1368 652 abandon();
Chris@1368 653 return;
Chris@1368 654 }
Chris@320 655
Chris@850 656 Transform primaryTransform = m_transforms[0];
Chris@850 657
Chris@497 658 while (!input->isReady() && !m_abandoned) {
Chris@877 659 cerr << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
Chris@497 660 usleep(500000);
Chris@320 661 }
Chris@497 662 if (m_abandoned) return;
Chris@320 663
Chris@1040 664 sv_samplerate_t sampleRate = input->getSampleRate();
Chris@320 665
Chris@930 666 int channelCount = input->getChannelCount();
Chris@930 667 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
Chris@1429 668 channelCount = 1;
Chris@320 669 }
Chris@320 670
Chris@320 671 float **buffers = new float*[channelCount];
Chris@930 672 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1429 673 buffers[ch] = new float[primaryTransform.getBlockSize() + 2];
Chris@320 674 }
Chris@320 675
Chris@930 676 int stepSize = primaryTransform.getStepSize();
Chris@930 677 int blockSize = primaryTransform.getBlockSize();
Chris@350 678
Chris@320 679 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 680 Vamp::Plugin::FrequencyDomain);
Chris@320 681 std::vector<FFTModel *> fftModels;
Chris@320 682
Chris@320 683 if (frequencyDomain) {
Chris@930 684 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 685 FFTModel *model = new FFTModel
Chris@350 686 (getConformingInput(),
Chris@350 687 channelCount == 1 ? m_input.getChannel() : ch,
Chris@850 688 primaryTransform.getWindowType(),
Chris@350 689 blockSize,
Chris@350 690 stepSize,
Chris@1090 691 blockSize);
Chris@1080 692 if (!model->isOK() || model->getError() != "") {
Chris@1080 693 QString err = model->getError();
Chris@320 694 delete model;
Chris@850 695 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 696 setCompletion(j, 100);
Chris@850 697 }
Chris@387 698 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@1080 699 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer: error is: " + err);
Chris@320 700 }
Chris@320 701 fftModels.push_back(model);
Chris@1080 702 cerr << "created model for channel " << ch << endl;
Chris@320 703 }
Chris@320 704 }
Chris@320 705
Chris@1040 706 sv_frame_t startFrame = m_input.getModel()->getStartFrame();
Chris@1040 707 sv_frame_t endFrame = m_input.getModel()->getEndFrame();
Chris@320 708
Chris@850 709 RealTime contextStartRT = primaryTransform.getStartTime();
Chris@850 710 RealTime contextDurationRT = primaryTransform.getDuration();
Chris@350 711
Chris@1040 712 sv_frame_t contextStart =
Chris@350 713 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 714
Chris@1040 715 sv_frame_t contextDuration =
Chris@350 716 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 717
Chris@320 718 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 719 contextStart = startFrame;
Chris@320 720 }
Chris@320 721
Chris@320 722 if (contextDuration == 0) {
Chris@320 723 contextDuration = endFrame - contextStart;
Chris@320 724 }
Chris@320 725 if (contextStart + contextDuration > endFrame) {
Chris@320 726 contextDuration = endFrame - contextStart;
Chris@320 727 }
Chris@320 728
Chris@1039 729 sv_frame_t blockFrame = contextStart;
Chris@320 730
Chris@320 731 long prevCompletion = 0;
Chris@320 732
Chris@850 733 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 734 setCompletion(j, 0);
Chris@850 735 }
Chris@320 736
Chris@556 737 float *reals = 0;
Chris@556 738 float *imaginaries = 0;
Chris@556 739 if (frequencyDomain) {
Chris@556 740 reals = new float[blockSize/2 + 1];
Chris@556 741 imaginaries = new float[blockSize/2 + 1];
Chris@556 742 }
Chris@556 743
Chris@678 744 QString error = "";
Chris@678 745
Chris@1372 746 try {
Chris@1372 747 while (!m_abandoned) {
Chris@320 748
Chris@1372 749 if (frequencyDomain) {
Chris@1372 750 if (blockFrame - int(blockSize)/2 >
Chris@1372 751 contextStart + contextDuration) break;
Chris@1372 752 } else {
Chris@1372 753 if (blockFrame >=
Chris@1372 754 contextStart + contextDuration) break;
Chris@1372 755 }
Chris@320 756
Chris@1429 757 // SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@1429 758 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@687 759 // << blockSize << endl;
Chris@320 760
Chris@1372 761 int completion = int
Chris@1372 762 ((((blockFrame - contextStart) / stepSize) * 99) /
Chris@1372 763 (contextDuration / stepSize + 1));
Chris@320 764
Chris@1372 765 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 766
Chris@1372 767 if (frequencyDomain) {
Chris@1372 768 for (int ch = 0; ch < channelCount; ++ch) {
Chris@1372 769 int column = int((blockFrame - startFrame) / stepSize);
Chris@1372 770 if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) {
Chris@1372 771 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 772 buffers[ch][i*2] = reals[i];
Chris@1372 773 buffers[ch][i*2+1] = imaginaries[i];
Chris@1372 774 }
Chris@1372 775 } else {
Chris@1372 776 for (int i = 0; i <= blockSize/2; ++i) {
Chris@1372 777 buffers[ch][i*2] = 0.f;
Chris@1372 778 buffers[ch][i*2+1] = 0.f;
Chris@1372 779 }
Chris@1372 780 }
Chris@1372 781 error = fftModels[ch]->getError();
Chris@1372 782 if (error != "") {
Chris@1372 783 SVCERR << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << endl;
Chris@1372 784 m_abandoned = true;
Chris@1372 785 m_message = error;
Chris@1372 786 break;
Chris@1008 787 }
Chris@1372 788 }
Chris@1372 789 } else {
Chris@1372 790 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@1372 791 }
Chris@1372 792
Chris@1372 793 if (m_abandoned) break;
Chris@1372 794
Chris@1372 795 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@1372 796 (buffers, RealTime::frame2RealTime(blockFrame, sampleRate).toVampRealTime());
Chris@1372 797
Chris@1372 798 if (m_abandoned) break;
Chris@1372 799
Chris@1372 800 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 801 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 802 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 803 addFeature(j, blockFrame, feature);
Chris@678 804 }
Chris@363 805 }
Chris@1372 806
Chris@1372 807 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@1372 808 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 809 setCompletion(j, completion);
Chris@1372 810 }
Chris@1372 811 prevCompletion = completion;
Chris@1372 812 }
Chris@1372 813
Chris@1372 814 blockFrame += stepSize;
Chris@1372 815
Chris@320 816 }
Chris@320 817
Chris@1372 818 if (!m_abandoned) {
Chris@1372 819 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@497 820
Chris@1372 821 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@1372 822 for (int fi = 0; fi < (int)features[m_outputNos[j]].size(); ++fi) {
Chris@1372 823 Vamp::Plugin::Feature feature = features[m_outputNos[j]][fi];
Chris@1372 824 addFeature(j, blockFrame, feature);
Chris@1372 825 }
Chris@850 826 }
Chris@850 827 }
Chris@1372 828 } catch (const std::exception &e) {
Chris@1372 829 SVCERR << "FeatureExtractionModelTransformer::run: Exception caught: "
Chris@1372 830 << e.what() << endl;
Chris@1372 831 m_abandoned = true;
Chris@1372 832 m_message = e.what();
Chris@497 833 }
Chris@320 834
Chris@850 835 for (int j = 0; j < (int)m_outputNos.size(); ++j) {
Chris@850 836 setCompletion(j, 100);
Chris@850 837 }
Chris@320 838
Chris@320 839 if (frequencyDomain) {
Chris@930 840 for (int ch = 0; ch < channelCount; ++ch) {
Chris@320 841 delete fftModels[ch];
Chris@320 842 }
Chris@556 843 delete[] reals;
Chris@556 844 delete[] imaginaries;
Chris@320 845 }
Chris@974 846
Chris@974 847 for (int ch = 0; ch < channelCount; ++ch) {
Chris@974 848 delete[] buffers[ch];
Chris@974 849 }
Chris@974 850 delete[] buffers;
Chris@1237 851
Chris@1237 852 deinitialise();
Chris@320 853 }
Chris@320 854
Chris@320 855 void
Chris@363 856 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@1039 857 sv_frame_t startFrame,
Chris@1039 858 sv_frame_t size,
Chris@363 859 float **buffers)
Chris@320 860 {
Chris@1039 861 sv_frame_t offset = 0;
Chris@320 862
Chris@320 863 if (startFrame < 0) {
Chris@363 864 for (int c = 0; c < channelCount; ++c) {
Chris@1039 865 for (sv_frame_t i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 866 buffers[c][i] = 0.0f;
Chris@363 867 }
Chris@320 868 }
Chris@320 869 offset = -startFrame;
Chris@320 870 size -= offset;
Chris@320 871 if (size <= 0) return;
Chris@320 872 startFrame = 0;
Chris@320 873 }
Chris@320 874
Chris@350 875 DenseTimeValueModel *input = getConformingInput();
Chris@350 876 if (!input) return;
Chris@363 877
Chris@1039 878 sv_frame_t got = 0;
Chris@350 879
Chris@363 880 if (channelCount == 1) {
Chris@363 881
Chris@1096 882 auto data = input->getData(m_input.getChannel(), startFrame, size);
Chris@1096 883 got = data.size();
Chris@1096 884
Chris@1096 885 copy(data.begin(), data.end(), buffers[0] + offset);
Chris@363 886
Chris@363 887 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 888 // use mean instead of sum, as plugin input
Chris@363 889 float cc = float(input->getChannelCount());
Chris@1096 890 for (sv_frame_t i = 0; i < got; ++i) {
Chris@363 891 buffers[0][i + offset] /= cc;
Chris@363 892 }
Chris@363 893 }
Chris@363 894
Chris@363 895 } else {
Chris@363 896
Chris@1096 897 auto data = input->getMultiChannelData(0, channelCount-1, startFrame, size);
Chris@1096 898 if (!data.empty()) {
Chris@1096 899 got = data[0].size();
Chris@1096 900 for (int c = 0; in_range_for(data, c); ++c) {
Chris@1096 901 copy(data[c].begin(), data[c].end(), buffers[c] + offset);
Chris@363 902 }
Chris@363 903 }
Chris@363 904 }
Chris@320 905
Chris@320 906 while (got < size) {
Chris@363 907 for (int c = 0; c < channelCount; ++c) {
Chris@363 908 buffers[c][got + offset] = 0.0;
Chris@363 909 }
Chris@320 910 ++got;
Chris@320 911 }
Chris@320 912 }
Chris@320 913
Chris@320 914 void
Chris@850 915 FeatureExtractionModelTransformer::addFeature(int n,
Chris@1039 916 sv_frame_t blockFrame,
Chris@850 917 const Vamp::Plugin::Feature &feature)
Chris@320 918 {
Chris@1040 919 sv_samplerate_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 920
Chris@843 921 // cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
Chris@712 922 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
Chris@712 923 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
Chris@712 924 // << feature.hasDuration << ", duration = " << feature.duration
Chris@843 925 // << endl;
Chris@320 926
Chris@1039 927 sv_frame_t frame = blockFrame;
Chris@320 928
Chris@849 929 if (m_descriptors[n]->sampleType ==
Chris@1429 930 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 931
Chris@1429 932 if (!feature.hasTimestamp) {
Chris@1429 933 SVDEBUG
Chris@1429 934 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@1429 935 << "Feature has variable sample rate but no timestamp!"
Chris@1429 936 << endl;
Chris@1429 937 return;
Chris@1429 938 } else {
Chris@1429 939 frame = RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@1429 940 }
Chris@320 941
Chris@1071 942 // cerr << "variable sample rate: timestamp = " << feature.timestamp
Chris@1071 943 // << " at input rate " << inputRate << " -> " << frame << endl;
Chris@1071 944
Chris@849 945 } else if (m_descriptors[n]->sampleType ==
Chris@1429 946 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 947
Chris@1071 948 sv_samplerate_t rate = m_descriptors[n]->sampleRate;
Chris@1071 949 if (rate <= 0.0) {
Chris@1071 950 rate = inputRate;
Chris@1071 951 }
Chris@1071 952
Chris@779 953 if (!feature.hasTimestamp) {
Chris@849 954 ++m_fixedRateFeatureNos[n];
Chris@779 955 } else {
Chris@779 956 RealTime ts(feature.timestamp.sec, feature.timestamp.nsec);
Chris@1071 957 m_fixedRateFeatureNos[n] = (int)lrint(ts.toDouble() * rate);
Chris@779 958 }
Chris@862 959
Chris@1071 960 // cerr << "m_fixedRateFeatureNo = " << m_fixedRateFeatureNos[n]
Chris@1071 961 // << ", m_descriptor->sampleRate = " << m_descriptors[n]->sampleRate
Chris@862 962 // << ", inputRate = " << inputRate
Chris@862 963 // << " giving frame = ";
Chris@1071 964 frame = lrint((double(m_fixedRateFeatureNos[n]) / rate) * inputRate);
Chris@1071 965 // cerr << frame << endl;
Chris@320 966 }
Chris@862 967
Chris@862 968 if (frame < 0) {
Chris@1264 969 SVDEBUG
Chris@862 970 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@862 971 << "Negative frame counts are not supported (frame = " << frame
Chris@862 972 << " from timestamp " << feature.timestamp
Chris@862 973 << "), dropping feature"
Chris@862 974 << endl;
Chris@862 975 return;
Chris@862 976 }
Chris@862 977
Chris@441 978 // Rather than repeat the complicated tests from the constructor
Chris@441 979 // to determine what sort of model we must be adding the features
Chris@441 980 // to, we instead test what sort of model the constructor decided
Chris@441 981 // to create.
Chris@320 982
Chris@849 983 if (isOutput<SparseOneDimensionalModel>(n)) {
Chris@441 984
Chris@441 985 SparseOneDimensionalModel *model =
Chris@849 986 getConformingOutput<SparseOneDimensionalModel>(n);
Chris@1429 987 if (!model) return;
Chris@350 988
Chris@441 989 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 990 (frame, feature.label.c_str()));
Chris@1429 991
Chris@849 992 } else if (isOutput<SparseTimeValueModel>(n)) {
Chris@320 993
Chris@1429 994 SparseTimeValueModel *model =
Chris@849 995 getConformingOutput<SparseTimeValueModel>(n);
Chris@1429 996 if (!model) return;
Chris@350 997
Chris@930 998 for (int i = 0; i < (int)feature.values.size(); ++i) {
Chris@454 999
Chris@454 1000 float value = feature.values[i];
Chris@454 1001
Chris@454 1002 QString label = feature.label.c_str();
Chris@454 1003 if (feature.values.size() > 1) {
Chris@454 1004 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 1005 }
Chris@454 1006
Chris@876 1007 SparseTimeValueModel *targetModel = model;
Chris@876 1008
Chris@876 1009 if (m_needAdditionalModels[n] && i > 0) {
Chris@876 1010 targetModel = getAdditionalModel(n, i);
Chris@876 1011 if (!targetModel) targetModel = model;
Chris@893 1012 // std::cerr << "adding point to model " << targetModel
Chris@893 1013 // << " for output " << n << " bin " << i << std::endl;
Chris@876 1014 }
Chris@876 1015
Chris@876 1016 targetModel->addPoint
Chris@876 1017 (SparseTimeValueModel::Point(frame, value, label));
Chris@454 1018 }
Chris@320 1019
Chris@849 1020 } else if (isOutput<FlexiNoteModel>(n) || isOutput<NoteModel>(n) || isOutput<RegionModel>(n)) { //GF: Added Note Model
Chris@320 1021
Chris@441 1022 int index = 0;
Chris@441 1023
Chris@441 1024 float value = 0.0;
Chris@930 1025 if ((int)feature.values.size() > index) {
Chris@441 1026 value = feature.values[index++];
Chris@441 1027 }
Chris@320 1028
Chris@1039 1029 sv_frame_t duration = 1;
Chris@441 1030 if (feature.hasDuration) {
Chris@1040 1031 duration = RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 1032 } else {
Chris@1039 1033 if (in_range_for(feature.values, index)) {
Chris@1039 1034 duration = lrintf(feature.values[index++]);
Chris@441 1035 }
Chris@441 1036 }
gyorgyf@786 1037
Chris@891 1038 if (isOutput<FlexiNoteModel>(n)) { // GF: added for flexi note model
gyorgyf@786 1039
gyorgyf@786 1040 float velocity = 100;
Chris@930 1041 if ((int)feature.values.size() > index) {
gyorgyf@786 1042 velocity = feature.values[index++];
gyorgyf@786 1043 }
gyorgyf@786 1044 if (velocity < 0) velocity = 127;
gyorgyf@786 1045 if (velocity > 127) velocity = 127;
gyorgyf@786 1046
Chris@849 1047 FlexiNoteModel *model = getConformingOutput<FlexiNoteModel>(n);
gyorgyf@786 1048 if (!model) return;
Chris@1039 1049 model->addPoint(FlexiNoteModel::Point(frame,
Chris@1039 1050 value, // value is pitch
Chris@1039 1051 duration,
Chris@1039 1052 velocity / 127.f,
Chris@1039 1053 feature.label.c_str()));
Chris@1429 1054 // GF: end -- added for flexi note model
Chris@849 1055 } else if (isOutput<NoteModel>(n)) {
Chris@320 1056
Chris@441 1057 float velocity = 100;
Chris@930 1058 if ((int)feature.values.size() > index) {
Chris@441 1059 velocity = feature.values[index++];
Chris@441 1060 }
Chris@441 1061 if (velocity < 0) velocity = 127;
Chris@441 1062 if (velocity > 127) velocity = 127;
Chris@320 1063
Chris@849 1064 NoteModel *model = getConformingOutput<NoteModel>(n);
Chris@441 1065 if (!model) return;
Chris@441 1066 model->addPoint(NoteModel::Point(frame, value, // value is pitch
Chris@1039 1067 duration,
Chris@441 1068 velocity / 127.f,
Chris@441 1069 feature.label.c_str()));
Chris@441 1070 } else {
gyorgyf@786 1071
Chris@849 1072 RegionModel *model = getConformingOutput<RegionModel>(n);
Chris@454 1073 if (!model) return;
Chris@454 1074
Chris@474 1075 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 1076
Chris@930 1077 for (int i = 0; i < (int)feature.values.size(); ++i) {
Chris@454 1078
Chris@454 1079 float value = feature.values[i];
Chris@454 1080
Chris@454 1081 QString label = feature.label.c_str();
Chris@454 1082 if (feature.values.size() > 1) {
Chris@454 1083 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 1084 }
Chris@454 1085
Chris@1039 1086 model->addPoint(RegionModel::Point(frame,
Chris@1039 1087 value,
Chris@1039 1088 duration,
Chris@454 1089 label));
Chris@454 1090 }
Chris@454 1091 } else {
Chris@454 1092
Chris@1039 1093 model->addPoint(RegionModel::Point(frame,
Chris@1039 1094 value,
Chris@1039 1095 duration,
Chris@441 1096 feature.label.c_str()));
Chris@454 1097 }
Chris@441 1098 }
Chris@1429 1099
Chris@849 1100 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) {
Chris@1429 1101
Chris@1429 1102 DenseThreeDimensionalModel::Column values = feature.values;
Chris@1429 1103
Chris@1429 1104 EditableDenseThreeDimensionalModel *model =
Chris@849 1105 getConformingOutput<EditableDenseThreeDimensionalModel>(n);
Chris@1429 1106 if (!model) return;
Chris@320 1107
Chris@889 1108 // cerr << "(note: model resolution = " << model->getResolution() << ")"
Chris@889 1109 // << endl;
Chris@889 1110
Chris@891 1111 if (!feature.hasTimestamp && m_fixedRateFeatureNos[n] >= 0) {
Chris@891 1112 model->setColumn(m_fixedRateFeatureNos[n], values);
Chris@889 1113 } else {
Chris@1039 1114 model->setColumn(int(frame / model->getResolution()), values);
Chris@889 1115 }
Chris@441 1116
Chris@441 1117 } else {
Chris@690 1118 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
Chris@320 1119 }
Chris@320 1120 }
Chris@320 1121
Chris@320 1122 void
Chris@850 1123 FeatureExtractionModelTransformer::setCompletion(int n, int completion)
Chris@320 1124 {
Chris@690 1125 // SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
Chris@687 1126 // << completion << ")" << endl;
Chris@320 1127
Chris@849 1128 if (isOutput<SparseOneDimensionalModel>(n)) {
Chris@320 1129
Chris@1429 1130 SparseOneDimensionalModel *model =
Chris@849 1131 getConformingOutput<SparseOneDimensionalModel>(n);
Chris@1429 1132 if (!model) return;
Chris@923 1133 if (model->isAbandoning()) abandon();
Chris@1429 1134 model->setCompletion(completion, true);
Chris@320 1135
Chris@849 1136 } else if (isOutput<SparseTimeValueModel>(n)) {
Chris@320 1137
Chris@1429 1138 SparseTimeValueModel *model =
Chris@849 1139 getConformingOutput<SparseTimeValueModel>(n);
Chris@1429 1140 if (!model) return;
Chris@923 1141 if (model->isAbandoning()) abandon();
Chris@1429 1142 model->setCompletion(completion, true);
Chris@320 1143
Chris@849 1144 } else if (isOutput<NoteModel>(n)) {
Chris@320 1145
Chris@1429 1146 NoteModel *model = getConformingOutput<NoteModel>(n);
Chris@1429 1147 if (!model) return;
Chris@923 1148 if (model->isAbandoning()) abandon();
Chris@1429 1149 model->setCompletion(completion, true);
Chris@1429 1150
Chris@923 1151 } else if (isOutput<FlexiNoteModel>(n)) {
gyorgyf@786 1152
Chris@1429 1153 FlexiNoteModel *model = getConformingOutput<FlexiNoteModel>(n);
Chris@1429 1154 if (!model) return;
Chris@923 1155 if (model->isAbandoning()) abandon();
Chris@1429 1156 model->setCompletion(completion, true);
Chris@320 1157
Chris@849 1158 } else if (isOutput<RegionModel>(n)) {
Chris@441 1159
Chris@1429 1160 RegionModel *model = getConformingOutput<RegionModel>(n);
Chris@1429 1161 if (!model) return;
Chris@923 1162 if (model->isAbandoning()) abandon();
Chris@1429 1163 model->setCompletion(completion, true);
Chris@441 1164
Chris@849 1165 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) {
Chris@320 1166
Chris@1429 1167 EditableDenseThreeDimensionalModel *model =
Chris@849 1168 getConformingOutput<EditableDenseThreeDimensionalModel>(n);
Chris@1429 1169 if (!model) return;
Chris@923 1170 if (model->isAbandoning()) abandon();
Chris@1429 1171 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 1172 }
Chris@320 1173 }
Chris@320 1174