annotate transform/FeatureExtractionModelTransformer.cpp @ 718:f3fd2988fc9b

Fix incorrect query structure for output type URIs. This led to some output RDF features being written with type URIs intended for different outputs. Also revert some SVDEBUGs to cerrs -- they are intended as user-visible errors or warnings rather than debug
author Chris Cannam
date Mon, 09 Jan 2012 16:28:54 +0000
parents 7f76499ef4f2
children eea8049df526 5295bdb58840
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@320 19 #include "plugin/PluginXml.h"
Chris@475 20 #include <vamp-hostsdk/Plugin.h>
Chris@320 21
Chris@320 22 #include "data/model/Model.h"
Chris@320 23 #include "base/Window.h"
Chris@387 24 #include "base/Exceptions.h"
Chris@320 25 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 26 #include "data/model/SparseTimeValueModel.h"
Chris@320 27 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 28 #include "data/model/DenseTimeValueModel.h"
Chris@320 29 #include "data/model/NoteModel.h"
Chris@441 30 #include "data/model/RegionModel.h"
Chris@320 31 #include "data/model/FFTModel.h"
Chris@320 32 #include "data/model/WaveFileModel.h"
Chris@558 33 #include "rdf/PluginRDFDescription.h"
Chris@320 34
Chris@350 35 #include "TransformFactory.h"
Chris@350 36
Chris@320 37 #include <iostream>
Chris@320 38
Chris@350 39 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@350 40 const Transform &transform) :
Chris@350 41 ModelTransformer(in, transform),
Chris@320 42 m_plugin(0),
Chris@320 43 m_descriptor(0),
Chris@320 44 m_outputFeatureNo(0)
Chris@320 45 {
Chris@690 46 // SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId << ", outputName " << m_transform.getOutput() << endl;
Chris@350 47
Chris@350 48 QString pluginId = transform.getPluginIdentifier();
Chris@320 49
Chris@320 50 FeatureExtractionPluginFactory *factory =
Chris@320 51 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@320 52
Chris@320 53 if (!factory) {
Chris@361 54 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@320 55 return;
Chris@320 56 }
Chris@320 57
Chris@350 58 DenseTimeValueModel *input = getConformingInput();
Chris@350 59 if (!input) {
Chris@361 60 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@350 61 return;
Chris@350 62 }
Chris@320 63
Chris@350 64 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 65 if (!m_plugin) {
Chris@361 66 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@320 67 return;
Chris@320 68 }
Chris@320 69
Chris@350 70 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@350 71 (m_transform, m_plugin);
Chris@343 72
Chris@350 73 TransformFactory::getInstance()->setPluginParameters
Chris@350 74 (m_transform, m_plugin);
Chris@320 75
Chris@320 76 size_t channelCount = input->getChannelCount();
Chris@320 77 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 78 channelCount = 1;
Chris@320 79 }
Chris@320 80 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@361 81 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 82 .arg(pluginId)
Chris@361 83 .arg(m_plugin->getMinChannelCount())
Chris@361 84 .arg(m_plugin->getMaxChannelCount())
Chris@361 85 .arg(input->getChannelCount());
Chris@320 86 return;
Chris@320 87 }
Chris@320 88
Chris@690 89 SVDEBUG << "Initialising feature extraction plugin with channels = "
Chris@350 90 << channelCount << ", step = " << m_transform.getStepSize()
Chris@687 91 << ", block = " << m_transform.getBlockSize() << endl;
Chris@320 92
Chris@320 93 if (!m_plugin->initialise(channelCount,
Chris@350 94 m_transform.getStepSize(),
Chris@350 95 m_transform.getBlockSize())) {
Chris@361 96
Chris@361 97 size_t pstep = m_transform.getStepSize();
Chris@361 98 size_t pblock = m_transform.getBlockSize();
Chris@361 99
Chris@361 100 m_transform.setStepSize(0);
Chris@361 101 m_transform.setBlockSize(0);
Chris@361 102 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@361 103 (m_transform, m_plugin);
Chris@361 104
Chris@361 105 if (m_transform.getStepSize() != pstep ||
Chris@361 106 m_transform.getBlockSize() != pblock) {
Chris@361 107
Chris@361 108 if (!m_plugin->initialise(channelCount,
Chris@361 109 m_transform.getStepSize(),
Chris@361 110 m_transform.getBlockSize())) {
Chris@361 111
Chris@361 112 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 113 return;
Chris@361 114
Chris@361 115 } else {
Chris@361 116
Chris@361 117 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 118 .arg(pluginId)
Chris@361 119 .arg(pstep)
Chris@361 120 .arg(pblock)
Chris@361 121 .arg(m_transform.getStepSize())
Chris@361 122 .arg(m_transform.getBlockSize());
Chris@361 123 }
Chris@361 124
Chris@361 125 } else {
Chris@361 126
Chris@361 127 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 128 return;
Chris@361 129 }
Chris@320 130 }
Chris@320 131
Chris@366 132 if (m_transform.getPluginVersion() != "") {
Chris@366 133 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@366 134 if (pv != m_transform.getPluginVersion()) {
Chris@366 135 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@366 136 .arg(m_transform.getPluginVersion())
Chris@366 137 .arg(pluginId)
Chris@366 138 .arg(pv);
Chris@366 139 if (m_message != "") {
Chris@366 140 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 141 } else {
Chris@366 142 m_message = vm;
Chris@366 143 }
Chris@366 144 }
Chris@366 145 }
Chris@366 146
Chris@320 147 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 148
Chris@320 149 if (outputs.empty()) {
Chris@361 150 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@320 151 return;
Chris@320 152 }
Chris@320 153
Chris@320 154 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@690 155 // SVDEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl;
Chris@350 156 if (m_transform.getOutput() == "" ||
Chris@350 157 outputs[i].identifier == m_transform.getOutput().toStdString()) {
Chris@320 158 m_outputFeatureNo = i;
Chris@441 159 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
Chris@320 160 break;
Chris@320 161 }
Chris@320 162 }
Chris@320 163
Chris@320 164 if (!m_descriptor) {
Chris@361 165 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@361 166 .arg(pluginId)
Chris@361 167 .arg(m_transform.getOutput());
Chris@320 168 return;
Chris@320 169 }
Chris@320 170
Chris@558 171 createOutputModel();
Chris@558 172 }
Chris@558 173
Chris@558 174 void
Chris@558 175 FeatureExtractionModelTransformer::createOutputModel()
Chris@558 176 {
Chris@558 177 DenseTimeValueModel *input = getConformingInput();
Chris@558 178
Chris@712 179 // std::cerr << "FeatureExtractionModelTransformer::createOutputModel: sample type " << m_descriptor->sampleType << ", rate " << m_descriptor->sampleRate << std::endl;
Chris@712 180
Chris@558 181 PluginRDFDescription description(m_transform.getPluginIdentifier());
Chris@558 182 QString outputId = m_transform.getOutput();
Chris@558 183
Chris@320 184 int binCount = 1;
Chris@320 185 float minValue = 0.0, maxValue = 0.0;
Chris@320 186 bool haveExtents = false;
Chris@320 187
Chris@320 188 if (m_descriptor->hasFixedBinCount) {
Chris@320 189 binCount = m_descriptor->binCount;
Chris@320 190 }
Chris@320 191
Chris@331 192 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@320 193 // << binCount << std::endl;
Chris@320 194
Chris@320 195 if (binCount > 0 && m_descriptor->hasKnownExtents) {
Chris@320 196 minValue = m_descriptor->minValue;
Chris@320 197 maxValue = m_descriptor->maxValue;
Chris@320 198 haveExtents = true;
Chris@320 199 }
Chris@320 200
Chris@350 201 size_t modelRate = input->getSampleRate();
Chris@320 202 size_t modelResolution = 1;
Chris@712 203
Chris@320 204 switch (m_descriptor->sampleType) {
Chris@320 205
Chris@320 206 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@320 207 if (m_descriptor->sampleRate != 0.0) {
Chris@320 208 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@320 209 }
Chris@320 210 break;
Chris@320 211
Chris@320 212 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@350 213 modelResolution = m_transform.getStepSize();
Chris@320 214 break;
Chris@320 215
Chris@320 216 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@451 217 //!!! SV doesn't actually support display of models that have
Chris@451 218 //!!! different underlying rates together -- so we always set
Chris@451 219 //!!! the model rate to be the input model's rate, and adjust
Chris@451 220 //!!! the resolution appropriately. We can't properly display
Chris@451 221 //!!! data with a higher resolution than the base model at all
Chris@451 222 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
Chris@451 223 if (m_descriptor->sampleRate > input->getSampleRate()) {
Chris@451 224 modelResolution = 1;
Chris@451 225 } else {
Chris@451 226 modelResolution = size_t(input->getSampleRate() /
Chris@451 227 m_descriptor->sampleRate);
Chris@451 228 }
Chris@320 229 break;
Chris@320 230 }
Chris@320 231
Chris@441 232 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 233
Chris@441 234 if (binCount == 0 &&
Chris@441 235 (preDurationPlugin || !m_descriptor->hasDuration)) {
Chris@320 236
Chris@445 237 // Anything with no value and no duration is an instant
Chris@445 238
Chris@320 239 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
Chris@320 240 false);
Chris@320 241
Chris@558 242 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@558 243 m_output->setRDFTypeURI(outputEventTypeURI);
Chris@558 244
Chris@441 245 } else if ((preDurationPlugin && binCount > 1 &&
Chris@441 246 (m_descriptor->sampleType ==
Chris@441 247 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@441 248 (!preDurationPlugin && m_descriptor->hasDuration)) {
Chris@441 249
Chris@441 250 // For plugins using the old v1 API without explicit duration,
Chris@441 251 // we treat anything that has multiple bins (i.e. that has the
Chris@441 252 // potential to have value and duration) and a variable sample
Chris@441 253 // rate as a note model, taking its values as pitch, duration
Chris@441 254 // and velocity (if present) respectively. This is the same
Chris@441 255 // behaviour as always applied by SV to these plugins in the
Chris@441 256 // past.
Chris@441 257
Chris@441 258 // For plugins with the newer API, we treat anything with
Chris@441 259 // duration as either a note model with pitch and velocity, or
Chris@441 260 // a region model.
Chris@441 261
Chris@441 262 // How do we know whether it's an interval or note model?
Chris@441 263 // What's the essential difference? Is a note model any
Chris@441 264 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 265 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 266 // note model always have velocity? This is a good question
Chris@441 267 // to be addressed by accompanying RDF, but for the moment we
Chris@441 268 // will do the following...
Chris@441 269
Chris@441 270 bool isNoteModel = false;
Chris@441 271
Chris@441 272 // Regions have only value (and duration -- we can't extract a
Chris@441 273 // region model from an old-style plugin that doesn't support
Chris@441 274 // duration)
Chris@441 275 if (binCount > 1) isNoteModel = true;
Chris@441 276
Chris@595 277 // Regions do not have units of Hz or MIDI things (a sweeping
Chris@595 278 // assumption!)
Chris@595 279 if (m_descriptor->unit == "Hz" ||
Chris@595 280 m_descriptor->unit.find("MIDI") != std::string::npos ||
Chris@595 281 m_descriptor->unit.find("midi") != std::string::npos) {
Chris@595 282 isNoteModel = true;
Chris@595 283 }
Chris@441 284
Chris@441 285 // If we had a "sparse 3D model", we would have the additional
Chris@441 286 // problem of determining whether to use that here (if bin
Chris@441 287 // count > 1). But we don't.
Chris@441 288
Chris@441 289 if (isNoteModel) {
Chris@441 290
Chris@441 291 NoteModel *model;
Chris@441 292 if (haveExtents) {
Chris@441 293 model = new NoteModel
Chris@441 294 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 295 } else {
Chris@441 296 model = new NoteModel
Chris@441 297 (modelRate, modelResolution, false);
Chris@441 298 }
Chris@441 299 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 300 m_output = model;
Chris@441 301
Chris@441 302 } else {
Chris@441 303
Chris@441 304 RegionModel *model;
Chris@441 305 if (haveExtents) {
Chris@441 306 model = new RegionModel
Chris@441 307 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 308 } else {
Chris@441 309 model = new RegionModel
Chris@441 310 (modelRate, modelResolution, false);
Chris@441 311 }
Chris@441 312 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 313 m_output = model;
Chris@441 314 }
Chris@441 315
Chris@558 316 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@558 317 m_output->setRDFTypeURI(outputEventTypeURI);
Chris@558 318
Chris@441 319 } else if (binCount == 1 ||
Chris@441 320 (m_descriptor->sampleType ==
Chris@441 321 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 322
Chris@441 323 // Anything that is not a 1D, note, or interval model and that
Chris@441 324 // has only one value per result must be a sparse time value
Chris@441 325 // model.
Chris@441 326
Chris@441 327 // Anything that is not a 1D, note, or interval model and that
Chris@441 328 // has a variable sample rate is also treated as a sparse time
Chris@441 329 // value model regardless of its bin count, because we lack a
Chris@441 330 // sparse 3D model.
Chris@320 331
Chris@320 332 SparseTimeValueModel *model;
Chris@320 333 if (haveExtents) {
Chris@320 334 model = new SparseTimeValueModel
Chris@320 335 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 336 } else {
Chris@320 337 model = new SparseTimeValueModel
Chris@320 338 (modelRate, modelResolution, false);
Chris@320 339 }
Chris@558 340
Chris@558 341 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 342 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
Chris@320 343
Chris@320 344 m_output = model;
Chris@320 345
Chris@558 346 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
Chris@558 347 m_output->setRDFTypeURI(outputEventTypeURI);
Chris@558 348
Chris@441 349 } else {
Chris@320 350
Chris@441 351 // Anything that is not a 1D, note, or interval model and that
Chris@441 352 // has a fixed sample rate and more than one value per result
Chris@441 353 // must be a dense 3D model.
Chris@320 354
Chris@320 355 EditableDenseThreeDimensionalModel *model =
Chris@320 356 new EditableDenseThreeDimensionalModel
Chris@535 357 (modelRate, modelResolution, binCount,
Chris@535 358 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 359 false);
Chris@320 360
Chris@320 361 if (!m_descriptor->binNames.empty()) {
Chris@320 362 std::vector<QString> names;
Chris@320 363 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
Chris@320 364 names.push_back(m_descriptor->binNames[i].c_str());
Chris@320 365 }
Chris@320 366 model->setBinNames(names);
Chris@320 367 }
Chris@320 368
Chris@320 369 m_output = model;
Chris@558 370
Chris@558 371 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
Chris@558 372 m_output->setRDFTypeURI(outputSignalTypeURI);
Chris@320 373 }
Chris@333 374
Chris@350 375 if (m_output) m_output->setSourceModel(input);
Chris@320 376 }
Chris@320 377
Chris@331 378 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 379 {
Chris@690 380 // SVDEBUG << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << endl;
Chris@320 381 delete m_plugin;
Chris@320 382 delete m_descriptor;
Chris@320 383 }
Chris@320 384
Chris@320 385 DenseTimeValueModel *
Chris@350 386 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 387 {
Chris@690 388 // SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
Chris@408 389
Chris@320 390 DenseTimeValueModel *dtvm =
Chris@320 391 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 392 if (!dtvm) {
Chris@690 393 SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
Chris@320 394 }
Chris@320 395 return dtvm;
Chris@320 396 }
Chris@320 397
Chris@320 398 void
Chris@331 399 FeatureExtractionModelTransformer::run()
Chris@320 400 {
Chris@350 401 DenseTimeValueModel *input = getConformingInput();
Chris@320 402 if (!input) return;
Chris@320 403
Chris@320 404 if (!m_output) return;
Chris@320 405
Chris@497 406 while (!input->isReady() && !m_abandoned) {
Chris@690 407 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
Chris@497 408 usleep(500000);
Chris@320 409 }
Chris@497 410 if (m_abandoned) return;
Chris@320 411
Chris@350 412 size_t sampleRate = input->getSampleRate();
Chris@320 413
Chris@320 414 size_t channelCount = input->getChannelCount();
Chris@320 415 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 416 channelCount = 1;
Chris@320 417 }
Chris@320 418
Chris@320 419 float **buffers = new float*[channelCount];
Chris@320 420 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 421 buffers[ch] = new float[m_transform.getBlockSize() + 2];
Chris@320 422 }
Chris@320 423
Chris@350 424 size_t stepSize = m_transform.getStepSize();
Chris@350 425 size_t blockSize = m_transform.getBlockSize();
Chris@350 426
Chris@320 427 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 428 Vamp::Plugin::FrequencyDomain);
Chris@320 429 std::vector<FFTModel *> fftModels;
Chris@320 430
Chris@320 431 if (frequencyDomain) {
Chris@320 432 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 433 FFTModel *model = new FFTModel
Chris@350 434 (getConformingInput(),
Chris@350 435 channelCount == 1 ? m_input.getChannel() : ch,
Chris@350 436 m_transform.getWindowType(),
Chris@350 437 blockSize,
Chris@350 438 stepSize,
Chris@350 439 blockSize,
Chris@334 440 false,
Chris@334 441 StorageAdviser::PrecisionCritical);
Chris@320 442 if (!model->isOK()) {
Chris@320 443 delete model;
Chris@320 444 setCompletion(100);
Chris@387 445 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@387 446 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
Chris@320 447 }
Chris@320 448 model->resume();
Chris@320 449 fftModels.push_back(model);
Chris@320 450 }
Chris@320 451 }
Chris@320 452
Chris@350 453 long startFrame = m_input.getModel()->getStartFrame();
Chris@350 454 long endFrame = m_input.getModel()->getEndFrame();
Chris@320 455
Chris@350 456 RealTime contextStartRT = m_transform.getStartTime();
Chris@350 457 RealTime contextDurationRT = m_transform.getDuration();
Chris@350 458
Chris@350 459 long contextStart =
Chris@350 460 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 461
Chris@350 462 long contextDuration =
Chris@350 463 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 464
Chris@320 465 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 466 contextStart = startFrame;
Chris@320 467 }
Chris@320 468
Chris@320 469 if (contextDuration == 0) {
Chris@320 470 contextDuration = endFrame - contextStart;
Chris@320 471 }
Chris@320 472 if (contextStart + contextDuration > endFrame) {
Chris@320 473 contextDuration = endFrame - contextStart;
Chris@320 474 }
Chris@320 475
Chris@320 476 long blockFrame = contextStart;
Chris@320 477
Chris@320 478 long prevCompletion = 0;
Chris@320 479
Chris@320 480 setCompletion(0);
Chris@320 481
Chris@556 482 float *reals = 0;
Chris@556 483 float *imaginaries = 0;
Chris@556 484 if (frequencyDomain) {
Chris@556 485 reals = new float[blockSize/2 + 1];
Chris@556 486 imaginaries = new float[blockSize/2 + 1];
Chris@556 487 }
Chris@556 488
Chris@678 489 QString error = "";
Chris@678 490
Chris@320 491 while (!m_abandoned) {
Chris@320 492
Chris@320 493 if (frequencyDomain) {
Chris@350 494 if (blockFrame - int(blockSize)/2 >
Chris@320 495 contextStart + contextDuration) break;
Chris@320 496 } else {
Chris@320 497 if (blockFrame >=
Chris@320 498 contextStart + contextDuration) break;
Chris@320 499 }
Chris@320 500
Chris@690 501 // SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@320 502 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@687 503 // << blockSize << endl;
Chris@320 504
Chris@320 505 long completion =
Chris@350 506 (((blockFrame - contextStart) / stepSize) * 99) /
Chris@557 507 (contextDuration / stepSize + 1);
Chris@320 508
Chris@350 509 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 510
Chris@363 511 if (frequencyDomain) {
Chris@363 512 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 513 int column = (blockFrame - startFrame) / stepSize;
Chris@556 514 fftModels[ch]->getValuesAt(column, reals, imaginaries);
Chris@350 515 for (size_t i = 0; i <= blockSize/2; ++i) {
Chris@556 516 buffers[ch][i*2] = reals[i];
Chris@556 517 buffers[ch][i*2+1] = imaginaries[i];
Chris@320 518 }
Chris@678 519 error = fftModels[ch]->getError();
Chris@678 520 if (error != "") {
Chris@686 521 std::cerr << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << std::endl;
Chris@678 522 m_abandoned = true;
Chris@678 523 m_message = error;
Chris@678 524 }
Chris@363 525 }
Chris@363 526 } else {
Chris@363 527 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@320 528 }
Chris@320 529
Chris@497 530 if (m_abandoned) break;
Chris@497 531
Chris@320 532 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@320 533 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@320 534
Chris@497 535 if (m_abandoned) break;
Chris@497 536
Chris@320 537 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@320 538 Vamp::Plugin::Feature feature =
Chris@320 539 features[m_outputFeatureNo][fi];
Chris@320 540 addFeature(blockFrame, feature);
Chris@320 541 }
Chris@320 542
Chris@320 543 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@320 544 setCompletion(completion);
Chris@320 545 prevCompletion = completion;
Chris@320 546 }
Chris@320 547
Chris@350 548 blockFrame += stepSize;
Chris@320 549 }
Chris@320 550
Chris@497 551 if (!m_abandoned) {
Chris@497 552 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@320 553
Chris@497 554 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@497 555 Vamp::Plugin::Feature feature =
Chris@497 556 features[m_outputFeatureNo][fi];
Chris@497 557 addFeature(blockFrame, feature);
Chris@497 558 }
Chris@497 559 }
Chris@320 560
Chris@497 561 setCompletion(100);
Chris@320 562
Chris@320 563 if (frequencyDomain) {
Chris@320 564 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 565 delete fftModels[ch];
Chris@320 566 }
Chris@556 567 delete[] reals;
Chris@556 568 delete[] imaginaries;
Chris@320 569 }
Chris@320 570 }
Chris@320 571
Chris@320 572 void
Chris@363 573 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@363 574 long startFrame, long size,
Chris@363 575 float **buffers)
Chris@320 576 {
Chris@320 577 long offset = 0;
Chris@320 578
Chris@320 579 if (startFrame < 0) {
Chris@363 580 for (int c = 0; c < channelCount; ++c) {
Chris@363 581 for (int i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 582 buffers[c][i] = 0.0f;
Chris@363 583 }
Chris@320 584 }
Chris@320 585 offset = -startFrame;
Chris@320 586 size -= offset;
Chris@320 587 if (size <= 0) return;
Chris@320 588 startFrame = 0;
Chris@320 589 }
Chris@320 590
Chris@350 591 DenseTimeValueModel *input = getConformingInput();
Chris@350 592 if (!input) return;
Chris@363 593
Chris@363 594 long got = 0;
Chris@350 595
Chris@363 596 if (channelCount == 1) {
Chris@363 597
Chris@363 598 got = input->getData(m_input.getChannel(), startFrame, size,
Chris@363 599 buffers[0] + offset);
Chris@363 600
Chris@363 601 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 602 // use mean instead of sum, as plugin input
Chris@363 603 float cc = float(input->getChannelCount());
Chris@363 604 for (long i = 0; i < size; ++i) {
Chris@363 605 buffers[0][i + offset] /= cc;
Chris@363 606 }
Chris@363 607 }
Chris@363 608
Chris@363 609 } else {
Chris@363 610
Chris@363 611 float **writebuf = buffers;
Chris@363 612 if (offset > 0) {
Chris@363 613 writebuf = new float *[channelCount];
Chris@363 614 for (int i = 0; i < channelCount; ++i) {
Chris@363 615 writebuf[i] = buffers[i] + offset;
Chris@363 616 }
Chris@363 617 }
Chris@363 618
Chris@363 619 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
Chris@363 620
Chris@363 621 if (writebuf != buffers) delete[] writebuf;
Chris@363 622 }
Chris@320 623
Chris@320 624 while (got < size) {
Chris@363 625 for (int c = 0; c < channelCount; ++c) {
Chris@363 626 buffers[c][got + offset] = 0.0;
Chris@363 627 }
Chris@320 628 ++got;
Chris@320 629 }
Chris@320 630 }
Chris@320 631
Chris@320 632 void
Chris@331 633 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
Chris@320 634 const Vamp::Plugin::Feature &feature)
Chris@320 635 {
Chris@350 636 size_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 637
Chris@712 638 // std::cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
Chris@712 639 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
Chris@712 640 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
Chris@712 641 // << feature.hasDuration << ", duration = " << feature.duration
Chris@712 642 // << std::endl;
Chris@320 643
Chris@320 644 int binCount = 1;
Chris@320 645 if (m_descriptor->hasFixedBinCount) {
Chris@320 646 binCount = m_descriptor->binCount;
Chris@320 647 }
Chris@320 648
Chris@320 649 size_t frame = blockFrame;
Chris@320 650
Chris@320 651 if (m_descriptor->sampleType ==
Chris@320 652 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 653
Chris@320 654 if (!feature.hasTimestamp) {
Chris@320 655 std::cerr
Chris@331 656 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@320 657 << "Feature has variable sample rate but no timestamp!"
Chris@320 658 << std::endl;
Chris@320 659 return;
Chris@320 660 } else {
Chris@320 661 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@320 662 }
Chris@320 663
Chris@320 664 } else if (m_descriptor->sampleType ==
Chris@320 665 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 666
Chris@320 667 if (feature.hasTimestamp) {
Chris@320 668 //!!! warning: sampleRate may be non-integral
Chris@320 669 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
Chris@451 670 //!!! see comment above when setting up modelResolution and modelRate
Chris@451 671 // lrintf(m_descriptor->sampleRate));
Chris@451 672 inputRate);
Chris@320 673 } else {
Chris@320 674 frame = m_output->getEndFrame();
Chris@320 675 }
Chris@320 676 }
Chris@320 677
Chris@441 678 // Rather than repeat the complicated tests from the constructor
Chris@441 679 // to determine what sort of model we must be adding the features
Chris@441 680 // to, we instead test what sort of model the constructor decided
Chris@441 681 // to create.
Chris@320 682
Chris@441 683 if (isOutput<SparseOneDimensionalModel>()) {
Chris@441 684
Chris@441 685 SparseOneDimensionalModel *model =
Chris@350 686 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 687 if (!model) return;
Chris@350 688
Chris@441 689 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 690 (frame, feature.label.c_str()));
Chris@320 691
Chris@441 692 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 693
Chris@350 694 SparseTimeValueModel *model =
Chris@350 695 getConformingOutput<SparseTimeValueModel>();
Chris@320 696 if (!model) return;
Chris@350 697
Chris@454 698 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 699
Chris@454 700 float value = feature.values[i];
Chris@454 701
Chris@454 702 QString label = feature.label.c_str();
Chris@454 703 if (feature.values.size() > 1) {
Chris@454 704 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 705 }
Chris@454 706
Chris@454 707 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
Chris@454 708 }
Chris@320 709
Chris@441 710 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
Chris@320 711
Chris@441 712 int index = 0;
Chris@441 713
Chris@441 714 float value = 0.0;
Chris@441 715 if (feature.values.size() > index) {
Chris@441 716 value = feature.values[index++];
Chris@441 717 }
Chris@320 718
Chris@320 719 float duration = 1;
Chris@441 720 if (feature.hasDuration) {
Chris@441 721 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 722 } else {
Chris@441 723 if (feature.values.size() > index) {
Chris@441 724 duration = feature.values[index++];
Chris@441 725 }
Chris@441 726 }
Chris@320 727
Chris@441 728 if (isOutput<NoteModel>()) {
Chris@320 729
Chris@441 730 float velocity = 100;
Chris@441 731 if (feature.values.size() > index) {
Chris@441 732 velocity = feature.values[index++];
Chris@441 733 }
Chris@441 734 if (velocity < 0) velocity = 127;
Chris@441 735 if (velocity > 127) velocity = 127;
Chris@320 736
Chris@441 737 NoteModel *model = getConformingOutput<NoteModel>();
Chris@441 738 if (!model) return;
Chris@441 739 model->addPoint(NoteModel::Point(frame, value, // value is pitch
Chris@441 740 lrintf(duration),
Chris@441 741 velocity / 127.f,
Chris@441 742 feature.label.c_str()));
Chris@441 743 } else {
Chris@441 744 RegionModel *model = getConformingOutput<RegionModel>();
Chris@454 745 if (!model) return;
Chris@454 746
Chris@474 747 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 748
Chris@454 749 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 750
Chris@454 751 float value = feature.values[i];
Chris@454 752
Chris@454 753 QString label = feature.label.c_str();
Chris@454 754 if (feature.values.size() > 1) {
Chris@454 755 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 756 }
Chris@454 757
Chris@454 758 model->addPoint(RegionModel::Point(frame, value,
Chris@454 759 lrintf(duration),
Chris@454 760 label));
Chris@454 761 }
Chris@454 762 } else {
Chris@454 763
Chris@441 764 model->addPoint(RegionModel::Point(frame, value,
Chris@441 765 lrintf(duration),
Chris@441 766 feature.label.c_str()));
Chris@454 767 }
Chris@441 768 }
Chris@320 769
Chris@441 770 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 771
Chris@533 772 DenseThreeDimensionalModel::Column values =
Chris@533 773 DenseThreeDimensionalModel::Column::fromStdVector(feature.values);
Chris@320 774
Chris@320 775 EditableDenseThreeDimensionalModel *model =
Chris@350 776 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 777 if (!model) return;
Chris@320 778
Chris@320 779 model->setColumn(frame / model->getResolution(), values);
Chris@441 780
Chris@441 781 } else {
Chris@690 782 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
Chris@320 783 }
Chris@320 784 }
Chris@320 785
Chris@320 786 void
Chris@331 787 FeatureExtractionModelTransformer::setCompletion(int completion)
Chris@320 788 {
Chris@320 789 int binCount = 1;
Chris@320 790 if (m_descriptor->hasFixedBinCount) {
Chris@320 791 binCount = m_descriptor->binCount;
Chris@320 792 }
Chris@320 793
Chris@690 794 // SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
Chris@687 795 // << completion << ")" << endl;
Chris@320 796
Chris@441 797 if (isOutput<SparseOneDimensionalModel>()) {
Chris@320 798
Chris@350 799 SparseOneDimensionalModel *model =
Chris@350 800 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 801 if (!model) return;
Chris@441 802 model->setCompletion(completion, true);
Chris@320 803
Chris@441 804 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 805
Chris@350 806 SparseTimeValueModel *model =
Chris@350 807 getConformingOutput<SparseTimeValueModel>();
Chris@320 808 if (!model) return;
Chris@441 809 model->setCompletion(completion, true);
Chris@320 810
Chris@441 811 } else if (isOutput<NoteModel>()) {
Chris@320 812
Chris@441 813 NoteModel *model = getConformingOutput<NoteModel>();
Chris@320 814 if (!model) return;
Chris@441 815 model->setCompletion(completion, true);
Chris@320 816
Chris@441 817 } else if (isOutput<RegionModel>()) {
Chris@441 818
Chris@441 819 RegionModel *model = getConformingOutput<RegionModel>();
Chris@441 820 if (!model) return;
Chris@441 821 model->setCompletion(completion, true);
Chris@441 822
Chris@441 823 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 824
Chris@320 825 EditableDenseThreeDimensionalModel *model =
Chris@350 826 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 827 if (!model) return;
Chris@350 828 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 829 }
Chris@320 830 }
Chris@320 831