annotate transform/FeatureExtractionModelTransformer.cpp @ 778:d373ced0e6e9

outputFeatureNo -> outputNo
author Chris Cannam
date Wed, 27 Mar 2013 15:07:27 +0000
parents eea8049df526
children 15cd67fcbdad
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@320 19 #include "plugin/PluginXml.h"
Chris@475 20 #include <vamp-hostsdk/Plugin.h>
Chris@320 21
Chris@320 22 #include "data/model/Model.h"
Chris@320 23 #include "base/Window.h"
Chris@387 24 #include "base/Exceptions.h"
Chris@320 25 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 26 #include "data/model/SparseTimeValueModel.h"
Chris@320 27 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 28 #include "data/model/DenseTimeValueModel.h"
Chris@320 29 #include "data/model/NoteModel.h"
Chris@441 30 #include "data/model/RegionModel.h"
Chris@320 31 #include "data/model/FFTModel.h"
Chris@320 32 #include "data/model/WaveFileModel.h"
Chris@558 33 #include "rdf/PluginRDFDescription.h"
Chris@320 34
Chris@350 35 #include "TransformFactory.h"
Chris@350 36
Chris@320 37 #include <iostream>
Chris@320 38
Chris@350 39 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@350 40 const Transform &transform) :
Chris@350 41 ModelTransformer(in, transform),
Chris@320 42 m_plugin(0),
Chris@320 43 m_descriptor(0),
Chris@778 44 m_outputNo(0)
Chris@320 45 {
Chris@690 46 // SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId << ", outputName " << m_transform.getOutput() << endl;
Chris@350 47
Chris@350 48 QString pluginId = transform.getPluginIdentifier();
Chris@320 49
Chris@320 50 FeatureExtractionPluginFactory *factory =
Chris@320 51 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@320 52
Chris@320 53 if (!factory) {
Chris@361 54 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@320 55 return;
Chris@320 56 }
Chris@320 57
Chris@350 58 DenseTimeValueModel *input = getConformingInput();
Chris@350 59 if (!input) {
Chris@361 60 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@350 61 return;
Chris@350 62 }
Chris@320 63
Chris@350 64 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 65 if (!m_plugin) {
Chris@361 66 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@320 67 return;
Chris@320 68 }
Chris@320 69
Chris@350 70 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@350 71 (m_transform, m_plugin);
Chris@343 72
Chris@350 73 TransformFactory::getInstance()->setPluginParameters
Chris@350 74 (m_transform, m_plugin);
Chris@320 75
Chris@320 76 size_t channelCount = input->getChannelCount();
Chris@320 77 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 78 channelCount = 1;
Chris@320 79 }
Chris@320 80 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@361 81 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 82 .arg(pluginId)
Chris@361 83 .arg(m_plugin->getMinChannelCount())
Chris@361 84 .arg(m_plugin->getMaxChannelCount())
Chris@361 85 .arg(input->getChannelCount());
Chris@320 86 return;
Chris@320 87 }
Chris@320 88
Chris@690 89 SVDEBUG << "Initialising feature extraction plugin with channels = "
Chris@350 90 << channelCount << ", step = " << m_transform.getStepSize()
Chris@687 91 << ", block = " << m_transform.getBlockSize() << endl;
Chris@320 92
Chris@320 93 if (!m_plugin->initialise(channelCount,
Chris@350 94 m_transform.getStepSize(),
Chris@350 95 m_transform.getBlockSize())) {
Chris@361 96
Chris@361 97 size_t pstep = m_transform.getStepSize();
Chris@361 98 size_t pblock = m_transform.getBlockSize();
Chris@361 99
Chris@361 100 m_transform.setStepSize(0);
Chris@361 101 m_transform.setBlockSize(0);
Chris@361 102 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@361 103 (m_transform, m_plugin);
Chris@361 104
Chris@361 105 if (m_transform.getStepSize() != pstep ||
Chris@361 106 m_transform.getBlockSize() != pblock) {
Chris@361 107
Chris@361 108 if (!m_plugin->initialise(channelCount,
Chris@361 109 m_transform.getStepSize(),
Chris@361 110 m_transform.getBlockSize())) {
Chris@361 111
Chris@361 112 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 113 return;
Chris@361 114
Chris@361 115 } else {
Chris@361 116
Chris@361 117 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 118 .arg(pluginId)
Chris@361 119 .arg(pstep)
Chris@361 120 .arg(pblock)
Chris@361 121 .arg(m_transform.getStepSize())
Chris@361 122 .arg(m_transform.getBlockSize());
Chris@361 123 }
Chris@361 124
Chris@361 125 } else {
Chris@361 126
Chris@361 127 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 128 return;
Chris@361 129 }
Chris@320 130 }
Chris@320 131
Chris@366 132 if (m_transform.getPluginVersion() != "") {
Chris@366 133 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@366 134 if (pv != m_transform.getPluginVersion()) {
Chris@366 135 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@366 136 .arg(m_transform.getPluginVersion())
Chris@366 137 .arg(pluginId)
Chris@366 138 .arg(pv);
Chris@366 139 if (m_message != "") {
Chris@366 140 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 141 } else {
Chris@366 142 m_message = vm;
Chris@366 143 }
Chris@366 144 }
Chris@366 145 }
Chris@366 146
Chris@320 147 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 148
Chris@320 149 if (outputs.empty()) {
Chris@361 150 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@320 151 return;
Chris@320 152 }
Chris@320 153
Chris@320 154 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@690 155 // SVDEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl;
Chris@350 156 if (m_transform.getOutput() == "" ||
Chris@350 157 outputs[i].identifier == m_transform.getOutput().toStdString()) {
Chris@778 158 m_outputNo = i;
Chris@441 159 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
Chris@320 160 break;
Chris@320 161 }
Chris@320 162 }
Chris@320 163
Chris@320 164 if (!m_descriptor) {
Chris@361 165 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@361 166 .arg(pluginId)
Chris@361 167 .arg(m_transform.getOutput());
Chris@320 168 return;
Chris@320 169 }
Chris@320 170
Chris@558 171 createOutputModel();
Chris@558 172 }
Chris@558 173
Chris@558 174 void
Chris@558 175 FeatureExtractionModelTransformer::createOutputModel()
Chris@558 176 {
Chris@558 177 DenseTimeValueModel *input = getConformingInput();
Chris@558 178
Chris@712 179 // std::cerr << "FeatureExtractionModelTransformer::createOutputModel: sample type " << m_descriptor->sampleType << ", rate " << m_descriptor->sampleRate << std::endl;
Chris@712 180
Chris@558 181 PluginRDFDescription description(m_transform.getPluginIdentifier());
Chris@558 182 QString outputId = m_transform.getOutput();
Chris@558 183
Chris@320 184 int binCount = 1;
Chris@320 185 float minValue = 0.0, maxValue = 0.0;
Chris@320 186 bool haveExtents = false;
Chris@320 187
Chris@320 188 if (m_descriptor->hasFixedBinCount) {
Chris@320 189 binCount = m_descriptor->binCount;
Chris@320 190 }
Chris@320 191
Chris@331 192 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@320 193 // << binCount << std::endl;
Chris@320 194
Chris@320 195 if (binCount > 0 && m_descriptor->hasKnownExtents) {
Chris@320 196 minValue = m_descriptor->minValue;
Chris@320 197 maxValue = m_descriptor->maxValue;
Chris@320 198 haveExtents = true;
Chris@320 199 }
Chris@320 200
Chris@350 201 size_t modelRate = input->getSampleRate();
Chris@320 202 size_t modelResolution = 1;
Chris@712 203
Chris@320 204 switch (m_descriptor->sampleType) {
Chris@320 205
Chris@320 206 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@320 207 if (m_descriptor->sampleRate != 0.0) {
Chris@320 208 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@320 209 }
Chris@320 210 break;
Chris@320 211
Chris@320 212 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@350 213 modelResolution = m_transform.getStepSize();
Chris@320 214 break;
Chris@320 215
Chris@320 216 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@451 217 //!!! SV doesn't actually support display of models that have
Chris@451 218 //!!! different underlying rates together -- so we always set
Chris@451 219 //!!! the model rate to be the input model's rate, and adjust
Chris@451 220 //!!! the resolution appropriately. We can't properly display
Chris@451 221 //!!! data with a higher resolution than the base model at all
Chris@451 222 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
Chris@451 223 if (m_descriptor->sampleRate > input->getSampleRate()) {
Chris@451 224 modelResolution = 1;
Chris@451 225 } else {
Chris@451 226 modelResolution = size_t(input->getSampleRate() /
Chris@451 227 m_descriptor->sampleRate);
Chris@451 228 }
Chris@320 229 break;
Chris@320 230 }
Chris@320 231
Chris@441 232 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 233
Chris@441 234 if (binCount == 0 &&
Chris@441 235 (preDurationPlugin || !m_descriptor->hasDuration)) {
Chris@320 236
Chris@445 237 // Anything with no value and no duration is an instant
Chris@445 238
Chris@320 239 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
Chris@320 240 false);
Chris@320 241
Chris@558 242 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@558 243 m_output->setRDFTypeURI(outputEventTypeURI);
Chris@558 244
Chris@441 245 } else if ((preDurationPlugin && binCount > 1 &&
Chris@441 246 (m_descriptor->sampleType ==
Chris@441 247 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@441 248 (!preDurationPlugin && m_descriptor->hasDuration)) {
Chris@441 249
Chris@441 250 // For plugins using the old v1 API without explicit duration,
Chris@441 251 // we treat anything that has multiple bins (i.e. that has the
Chris@441 252 // potential to have value and duration) and a variable sample
Chris@441 253 // rate as a note model, taking its values as pitch, duration
Chris@441 254 // and velocity (if present) respectively. This is the same
Chris@441 255 // behaviour as always applied by SV to these plugins in the
Chris@441 256 // past.
Chris@441 257
Chris@441 258 // For plugins with the newer API, we treat anything with
Chris@441 259 // duration as either a note model with pitch and velocity, or
Chris@441 260 // a region model.
Chris@441 261
Chris@441 262 // How do we know whether it's an interval or note model?
Chris@441 263 // What's the essential difference? Is a note model any
Chris@441 264 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 265 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 266 // note model always have velocity? This is a good question
Chris@441 267 // to be addressed by accompanying RDF, but for the moment we
Chris@441 268 // will do the following...
Chris@441 269
Chris@441 270 bool isNoteModel = false;
Chris@441 271
Chris@441 272 // Regions have only value (and duration -- we can't extract a
Chris@441 273 // region model from an old-style plugin that doesn't support
Chris@441 274 // duration)
Chris@441 275 if (binCount > 1) isNoteModel = true;
Chris@441 276
Chris@595 277 // Regions do not have units of Hz or MIDI things (a sweeping
Chris@595 278 // assumption!)
Chris@595 279 if (m_descriptor->unit == "Hz" ||
Chris@595 280 m_descriptor->unit.find("MIDI") != std::string::npos ||
Chris@595 281 m_descriptor->unit.find("midi") != std::string::npos) {
Chris@595 282 isNoteModel = true;
Chris@595 283 }
Chris@441 284
Chris@441 285 // If we had a "sparse 3D model", we would have the additional
Chris@441 286 // problem of determining whether to use that here (if bin
Chris@441 287 // count > 1). But we don't.
Chris@441 288
Chris@441 289 if (isNoteModel) {
Chris@441 290
Chris@441 291 NoteModel *model;
Chris@441 292 if (haveExtents) {
Chris@441 293 model = new NoteModel
Chris@441 294 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 295 } else {
Chris@441 296 model = new NoteModel
Chris@441 297 (modelRate, modelResolution, false);
Chris@441 298 }
Chris@441 299 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 300 m_output = model;
Chris@441 301
Chris@441 302 } else {
Chris@441 303
Chris@441 304 RegionModel *model;
Chris@441 305 if (haveExtents) {
Chris@441 306 model = new RegionModel
Chris@441 307 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 308 } else {
Chris@441 309 model = new RegionModel
Chris@441 310 (modelRate, modelResolution, false);
Chris@441 311 }
Chris@441 312 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 313 m_output = model;
Chris@441 314 }
Chris@441 315
Chris@558 316 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@558 317 m_output->setRDFTypeURI(outputEventTypeURI);
Chris@558 318
Chris@441 319 } else if (binCount == 1 ||
Chris@441 320 (m_descriptor->sampleType ==
Chris@441 321 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 322
Chris@441 323 // Anything that is not a 1D, note, or interval model and that
Chris@441 324 // has only one value per result must be a sparse time value
Chris@441 325 // model.
Chris@441 326
Chris@441 327 // Anything that is not a 1D, note, or interval model and that
Chris@441 328 // has a variable sample rate is also treated as a sparse time
Chris@441 329 // value model regardless of its bin count, because we lack a
Chris@441 330 // sparse 3D model.
Chris@320 331
Chris@320 332 SparseTimeValueModel *model;
Chris@320 333 if (haveExtents) {
Chris@320 334 model = new SparseTimeValueModel
Chris@320 335 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 336 } else {
Chris@320 337 model = new SparseTimeValueModel
Chris@320 338 (modelRate, modelResolution, false);
Chris@320 339 }
Chris@558 340
Chris@558 341 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@778 342 model->setScaleUnits(outputs[m_outputNo].unit.c_str());
Chris@320 343
Chris@320 344 m_output = model;
Chris@320 345
Chris@558 346 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@558 347 m_output->setRDFTypeURI(outputEventTypeURI);
Chris@558 348
Chris@441 349 } else {
Chris@320 350
Chris@441 351 // Anything that is not a 1D, note, or interval model and that
Chris@441 352 // has a fixed sample rate and more than one value per result
Chris@441 353 // must be a dense 3D model.
Chris@320 354
Chris@320 355 EditableDenseThreeDimensionalModel *model =
Chris@320 356 new EditableDenseThreeDimensionalModel
Chris@535 357 (modelRate, modelResolution, binCount,
Chris@535 358 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 359 false);
Chris@320 360
Chris@320 361 if (!m_descriptor->binNames.empty()) {
Chris@320 362 std::vector<QString> names;
Chris@320 363 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
Chris@320 364 names.push_back(m_descriptor->binNames[i].c_str());
Chris@320 365 }
Chris@320 366 model->setBinNames(names);
Chris@320 367 }
Chris@320 368
Chris@320 369 m_output = model;
Chris@558 370
Chris@558 371 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
Chris@558 372 m_output->setRDFTypeURI(outputSignalTypeURI);
Chris@320 373 }
Chris@333 374
Chris@350 375 if (m_output) m_output->setSourceModel(input);
Chris@320 376 }
Chris@320 377
Chris@331 378 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 379 {
Chris@690 380 // SVDEBUG << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << endl;
Chris@320 381 delete m_plugin;
Chris@320 382 delete m_descriptor;
Chris@320 383 }
Chris@320 384
Chris@320 385 DenseTimeValueModel *
Chris@350 386 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 387 {
Chris@690 388 // SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
Chris@408 389
Chris@320 390 DenseTimeValueModel *dtvm =
Chris@320 391 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 392 if (!dtvm) {
Chris@690 393 SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
Chris@320 394 }
Chris@320 395 return dtvm;
Chris@320 396 }
Chris@320 397
Chris@320 398 void
Chris@331 399 FeatureExtractionModelTransformer::run()
Chris@320 400 {
Chris@350 401 DenseTimeValueModel *input = getConformingInput();
Chris@320 402 if (!input) return;
Chris@320 403
Chris@320 404 if (!m_output) return;
Chris@320 405
Chris@497 406 while (!input->isReady() && !m_abandoned) {
Chris@690 407 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
Chris@497 408 usleep(500000);
Chris@320 409 }
Chris@497 410 if (m_abandoned) return;
Chris@320 411
Chris@350 412 size_t sampleRate = input->getSampleRate();
Chris@320 413
Chris@320 414 size_t channelCount = input->getChannelCount();
Chris@320 415 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 416 channelCount = 1;
Chris@320 417 }
Chris@320 418
Chris@320 419 float **buffers = new float*[channelCount];
Chris@320 420 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 421 buffers[ch] = new float[m_transform.getBlockSize() + 2];
Chris@320 422 }
Chris@320 423
Chris@350 424 size_t stepSize = m_transform.getStepSize();
Chris@350 425 size_t blockSize = m_transform.getBlockSize();
Chris@350 426
Chris@320 427 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 428 Vamp::Plugin::FrequencyDomain);
Chris@320 429 std::vector<FFTModel *> fftModels;
Chris@320 430
Chris@320 431 if (frequencyDomain) {
Chris@320 432 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 433 FFTModel *model = new FFTModel
Chris@350 434 (getConformingInput(),
Chris@350 435 channelCount == 1 ? m_input.getChannel() : ch,
Chris@350 436 m_transform.getWindowType(),
Chris@350 437 blockSize,
Chris@350 438 stepSize,
Chris@350 439 blockSize,
Chris@334 440 false,
Chris@334 441 StorageAdviser::PrecisionCritical);
Chris@320 442 if (!model->isOK()) {
Chris@320 443 delete model;
Chris@320 444 setCompletion(100);
Chris@387 445 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@387 446 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
Chris@320 447 }
Chris@320 448 model->resume();
Chris@320 449 fftModels.push_back(model);
Chris@320 450 }
Chris@320 451 }
Chris@320 452
Chris@350 453 long startFrame = m_input.getModel()->getStartFrame();
Chris@350 454 long endFrame = m_input.getModel()->getEndFrame();
Chris@320 455
Chris@350 456 RealTime contextStartRT = m_transform.getStartTime();
Chris@350 457 RealTime contextDurationRT = m_transform.getDuration();
Chris@350 458
Chris@350 459 long contextStart =
Chris@350 460 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 461
Chris@350 462 long contextDuration =
Chris@350 463 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 464
Chris@320 465 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 466 contextStart = startFrame;
Chris@320 467 }
Chris@320 468
Chris@320 469 if (contextDuration == 0) {
Chris@320 470 contextDuration = endFrame - contextStart;
Chris@320 471 }
Chris@320 472 if (contextStart + contextDuration > endFrame) {
Chris@320 473 contextDuration = endFrame - contextStart;
Chris@320 474 }
Chris@320 475
Chris@320 476 long blockFrame = contextStart;
Chris@320 477
Chris@320 478 long prevCompletion = 0;
Chris@320 479
Chris@320 480 setCompletion(0);
Chris@320 481
Chris@556 482 float *reals = 0;
Chris@556 483 float *imaginaries = 0;
Chris@556 484 if (frequencyDomain) {
Chris@556 485 reals = new float[blockSize/2 + 1];
Chris@556 486 imaginaries = new float[blockSize/2 + 1];
Chris@556 487 }
Chris@556 488
Chris@678 489 QString error = "";
Chris@678 490
Chris@320 491 while (!m_abandoned) {
Chris@320 492
Chris@320 493 if (frequencyDomain) {
Chris@350 494 if (blockFrame - int(blockSize)/2 >
Chris@320 495 contextStart + contextDuration) break;
Chris@320 496 } else {
Chris@320 497 if (blockFrame >=
Chris@320 498 contextStart + contextDuration) break;
Chris@320 499 }
Chris@320 500
Chris@690 501 // SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@320 502 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@687 503 // << blockSize << endl;
Chris@320 504
Chris@320 505 long completion =
Chris@350 506 (((blockFrame - contextStart) / stepSize) * 99) /
Chris@557 507 (contextDuration / stepSize + 1);
Chris@320 508
Chris@350 509 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 510
Chris@363 511 if (frequencyDomain) {
Chris@363 512 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 513 int column = (blockFrame - startFrame) / stepSize;
Chris@556 514 fftModels[ch]->getValuesAt(column, reals, imaginaries);
Chris@350 515 for (size_t i = 0; i <= blockSize/2; ++i) {
Chris@556 516 buffers[ch][i*2] = reals[i];
Chris@556 517 buffers[ch][i*2+1] = imaginaries[i];
Chris@320 518 }
Chris@678 519 error = fftModels[ch]->getError();
Chris@678 520 if (error != "") {
Chris@686 521 std::cerr << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << std::endl;
Chris@678 522 m_abandoned = true;
Chris@678 523 m_message = error;
Chris@678 524 }
Chris@363 525 }
Chris@363 526 } else {
Chris@363 527 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@320 528 }
Chris@320 529
Chris@497 530 if (m_abandoned) break;
Chris@497 531
Chris@320 532 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@320 533 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@320 534
Chris@497 535 if (m_abandoned) break;
Chris@497 536
Chris@778 537 for (size_t fi = 0; fi < features[m_outputNo].size(); ++fi) {
Chris@778 538 Vamp::Plugin::Feature feature = features[m_outputNo][fi];
Chris@320 539 addFeature(blockFrame, feature);
Chris@320 540 }
Chris@320 541
Chris@320 542 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@320 543 setCompletion(completion);
Chris@320 544 prevCompletion = completion;
Chris@320 545 }
Chris@320 546
Chris@350 547 blockFrame += stepSize;
Chris@320 548 }
Chris@320 549
Chris@497 550 if (!m_abandoned) {
Chris@497 551 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@320 552
Chris@778 553 for (size_t fi = 0; fi < features[m_outputNo].size(); ++fi) {
Chris@778 554 Vamp::Plugin::Feature feature = features[m_outputNo][fi];
Chris@497 555 addFeature(blockFrame, feature);
Chris@497 556 }
Chris@497 557 }
Chris@320 558
Chris@497 559 setCompletion(100);
Chris@320 560
Chris@320 561 if (frequencyDomain) {
Chris@320 562 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 563 delete fftModels[ch];
Chris@320 564 }
Chris@556 565 delete[] reals;
Chris@556 566 delete[] imaginaries;
Chris@320 567 }
Chris@320 568 }
Chris@320 569
Chris@320 570 void
Chris@363 571 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@363 572 long startFrame, long size,
Chris@363 573 float **buffers)
Chris@320 574 {
Chris@320 575 long offset = 0;
Chris@320 576
Chris@320 577 if (startFrame < 0) {
Chris@363 578 for (int c = 0; c < channelCount; ++c) {
Chris@363 579 for (int i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 580 buffers[c][i] = 0.0f;
Chris@363 581 }
Chris@320 582 }
Chris@320 583 offset = -startFrame;
Chris@320 584 size -= offset;
Chris@320 585 if (size <= 0) return;
Chris@320 586 startFrame = 0;
Chris@320 587 }
Chris@320 588
Chris@350 589 DenseTimeValueModel *input = getConformingInput();
Chris@350 590 if (!input) return;
Chris@363 591
Chris@363 592 long got = 0;
Chris@350 593
Chris@363 594 if (channelCount == 1) {
Chris@363 595
Chris@363 596 got = input->getData(m_input.getChannel(), startFrame, size,
Chris@363 597 buffers[0] + offset);
Chris@363 598
Chris@363 599 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 600 // use mean instead of sum, as plugin input
Chris@363 601 float cc = float(input->getChannelCount());
Chris@363 602 for (long i = 0; i < size; ++i) {
Chris@363 603 buffers[0][i + offset] /= cc;
Chris@363 604 }
Chris@363 605 }
Chris@363 606
Chris@363 607 } else {
Chris@363 608
Chris@363 609 float **writebuf = buffers;
Chris@363 610 if (offset > 0) {
Chris@363 611 writebuf = new float *[channelCount];
Chris@363 612 for (int i = 0; i < channelCount; ++i) {
Chris@363 613 writebuf[i] = buffers[i] + offset;
Chris@363 614 }
Chris@363 615 }
Chris@363 616
Chris@363 617 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
Chris@363 618
Chris@363 619 if (writebuf != buffers) delete[] writebuf;
Chris@363 620 }
Chris@320 621
Chris@320 622 while (got < size) {
Chris@363 623 for (int c = 0; c < channelCount; ++c) {
Chris@363 624 buffers[c][got + offset] = 0.0;
Chris@363 625 }
Chris@320 626 ++got;
Chris@320 627 }
Chris@320 628 }
Chris@320 629
Chris@320 630 void
Chris@331 631 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
Chris@320 632 const Vamp::Plugin::Feature &feature)
Chris@320 633 {
Chris@350 634 size_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 635
Chris@712 636 // std::cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
Chris@712 637 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
Chris@712 638 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
Chris@712 639 // << feature.hasDuration << ", duration = " << feature.duration
Chris@712 640 // << std::endl;
Chris@320 641
Chris@320 642 int binCount = 1;
Chris@320 643 if (m_descriptor->hasFixedBinCount) {
Chris@320 644 binCount = m_descriptor->binCount;
Chris@320 645 }
Chris@320 646
Chris@320 647 size_t frame = blockFrame;
Chris@320 648
Chris@320 649 if (m_descriptor->sampleType ==
Chris@320 650 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 651
Chris@320 652 if (!feature.hasTimestamp) {
Chris@320 653 std::cerr
Chris@331 654 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@320 655 << "Feature has variable sample rate but no timestamp!"
Chris@320 656 << std::endl;
Chris@320 657 return;
Chris@320 658 } else {
Chris@320 659 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@320 660 }
Chris@320 661
Chris@320 662 } else if (m_descriptor->sampleType ==
Chris@320 663 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 664
Chris@320 665 if (feature.hasTimestamp) {
Chris@320 666 //!!! warning: sampleRate may be non-integral
Chris@320 667 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
Chris@451 668 //!!! see comment above when setting up modelResolution and modelRate
Chris@451 669 // lrintf(m_descriptor->sampleRate));
Chris@451 670 inputRate);
Chris@320 671 } else {
Chris@320 672 frame = m_output->getEndFrame();
Chris@320 673 }
Chris@777 674
Chris@777 675 // std::cerr << "Feature hasTimestamp = " << feature.hasTimestamp << ", timestamp = " << feature.timestamp << ", frame works out to " << frame << std::endl;
Chris@777 676
Chris@320 677 }
Chris@320 678
Chris@441 679 // Rather than repeat the complicated tests from the constructor
Chris@441 680 // to determine what sort of model we must be adding the features
Chris@441 681 // to, we instead test what sort of model the constructor decided
Chris@441 682 // to create.
Chris@320 683
Chris@441 684 if (isOutput<SparseOneDimensionalModel>()) {
Chris@441 685
Chris@441 686 SparseOneDimensionalModel *model =
Chris@350 687 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 688 if (!model) return;
Chris@350 689
Chris@441 690 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 691 (frame, feature.label.c_str()));
Chris@320 692
Chris@441 693 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 694
Chris@350 695 SparseTimeValueModel *model =
Chris@350 696 getConformingOutput<SparseTimeValueModel>();
Chris@320 697 if (!model) return;
Chris@350 698
Chris@454 699 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 700
Chris@454 701 float value = feature.values[i];
Chris@454 702
Chris@454 703 QString label = feature.label.c_str();
Chris@454 704 if (feature.values.size() > 1) {
Chris@454 705 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 706 }
Chris@454 707
Chris@777 708 std::cerr << "Adding point at " << frame << " with value " << value << " and label " << label << std::endl;
Chris@777 709
Chris@454 710 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
Chris@454 711 }
Chris@320 712
Chris@441 713 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
Chris@320 714
Chris@441 715 int index = 0;
Chris@441 716
Chris@441 717 float value = 0.0;
Chris@441 718 if (feature.values.size() > index) {
Chris@441 719 value = feature.values[index++];
Chris@441 720 }
Chris@320 721
Chris@320 722 float duration = 1;
Chris@441 723 if (feature.hasDuration) {
Chris@441 724 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 725 } else {
Chris@441 726 if (feature.values.size() > index) {
Chris@441 727 duration = feature.values[index++];
Chris@441 728 }
Chris@441 729 }
Chris@320 730
Chris@441 731 if (isOutput<NoteModel>()) {
Chris@320 732
Chris@441 733 float velocity = 100;
Chris@441 734 if (feature.values.size() > index) {
Chris@441 735 velocity = feature.values[index++];
Chris@441 736 }
Chris@441 737 if (velocity < 0) velocity = 127;
Chris@441 738 if (velocity > 127) velocity = 127;
Chris@320 739
Chris@441 740 NoteModel *model = getConformingOutput<NoteModel>();
Chris@441 741 if (!model) return;
Chris@441 742 model->addPoint(NoteModel::Point(frame, value, // value is pitch
Chris@441 743 lrintf(duration),
Chris@441 744 velocity / 127.f,
Chris@441 745 feature.label.c_str()));
Chris@441 746 } else {
Chris@441 747 RegionModel *model = getConformingOutput<RegionModel>();
Chris@454 748 if (!model) return;
Chris@454 749
Chris@474 750 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 751
Chris@454 752 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 753
Chris@454 754 float value = feature.values[i];
Chris@454 755
Chris@454 756 QString label = feature.label.c_str();
Chris@454 757 if (feature.values.size() > 1) {
Chris@454 758 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 759 }
Chris@454 760
Chris@454 761 model->addPoint(RegionModel::Point(frame, value,
Chris@454 762 lrintf(duration),
Chris@454 763 label));
Chris@454 764 }
Chris@454 765 } else {
Chris@454 766
Chris@441 767 model->addPoint(RegionModel::Point(frame, value,
Chris@441 768 lrintf(duration),
Chris@441 769 feature.label.c_str()));
Chris@454 770 }
Chris@441 771 }
Chris@320 772
Chris@441 773 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 774
Chris@533 775 DenseThreeDimensionalModel::Column values =
Chris@533 776 DenseThreeDimensionalModel::Column::fromStdVector(feature.values);
Chris@320 777
Chris@320 778 EditableDenseThreeDimensionalModel *model =
Chris@350 779 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 780 if (!model) return;
Chris@320 781
Chris@320 782 model->setColumn(frame / model->getResolution(), values);
Chris@441 783
Chris@441 784 } else {
Chris@690 785 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
Chris@320 786 }
Chris@320 787 }
Chris@320 788
Chris@320 789 void
Chris@331 790 FeatureExtractionModelTransformer::setCompletion(int completion)
Chris@320 791 {
Chris@320 792 int binCount = 1;
Chris@320 793 if (m_descriptor->hasFixedBinCount) {
Chris@320 794 binCount = m_descriptor->binCount;
Chris@320 795 }
Chris@320 796
Chris@690 797 // SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
Chris@687 798 // << completion << ")" << endl;
Chris@320 799
Chris@441 800 if (isOutput<SparseOneDimensionalModel>()) {
Chris@320 801
Chris@350 802 SparseOneDimensionalModel *model =
Chris@350 803 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 804 if (!model) return;
Chris@441 805 model->setCompletion(completion, true);
Chris@320 806
Chris@441 807 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 808
Chris@350 809 SparseTimeValueModel *model =
Chris@350 810 getConformingOutput<SparseTimeValueModel>();
Chris@320 811 if (!model) return;
Chris@441 812 model->setCompletion(completion, true);
Chris@320 813
Chris@441 814 } else if (isOutput<NoteModel>()) {
Chris@320 815
Chris@441 816 NoteModel *model = getConformingOutput<NoteModel>();
Chris@320 817 if (!model) return;
Chris@441 818 model->setCompletion(completion, true);
Chris@320 819
Chris@441 820 } else if (isOutput<RegionModel>()) {
Chris@441 821
Chris@441 822 RegionModel *model = getConformingOutput<RegionModel>();
Chris@441 823 if (!model) return;
Chris@441 824 model->setCompletion(completion, true);
Chris@441 825
Chris@441 826 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 827
Chris@320 828 EditableDenseThreeDimensionalModel *model =
Chris@350 829 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 830 if (!model) return;
Chris@350 831 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 832 }
Chris@320 833 }
Chris@320 834