annotate transform/FeatureExtractionModelTransformer.cpp @ 557:a40023bebd15

* Fix crash when processing a file that is shorter than a single block
author Chris Cannam
date Tue, 10 Feb 2009 17:24:30 +0000
parents 53e5dc8439e7
children 1d7ebc05157e
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@320 19 #include "plugin/PluginXml.h"
Chris@475 20 #include <vamp-hostsdk/Plugin.h>
Chris@320 21
Chris@320 22 #include "data/model/Model.h"
Chris@320 23 #include "base/Window.h"
Chris@387 24 #include "base/Exceptions.h"
Chris@320 25 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 26 #include "data/model/SparseTimeValueModel.h"
Chris@320 27 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 28 #include "data/model/DenseTimeValueModel.h"
Chris@320 29 #include "data/model/NoteModel.h"
Chris@441 30 #include "data/model/RegionModel.h"
Chris@320 31 #include "data/model/FFTModel.h"
Chris@320 32 #include "data/model/WaveFileModel.h"
Chris@320 33
Chris@350 34 #include "TransformFactory.h"
Chris@350 35
Chris@320 36 #include <iostream>
Chris@320 37
Chris@350 38 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@350 39 const Transform &transform) :
Chris@350 40 ModelTransformer(in, transform),
Chris@320 41 m_plugin(0),
Chris@320 42 m_descriptor(0),
Chris@320 43 m_outputFeatureNo(0)
Chris@320 44 {
Chris@350 45 // std::cerr << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId.toStdString() << ", outputName " << m_transform.getOutput().toStdString() << std::endl;
Chris@350 46
Chris@350 47 QString pluginId = transform.getPluginIdentifier();
Chris@320 48
Chris@320 49 FeatureExtractionPluginFactory *factory =
Chris@320 50 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@320 51
Chris@320 52 if (!factory) {
Chris@361 53 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@320 54 return;
Chris@320 55 }
Chris@320 56
Chris@350 57 DenseTimeValueModel *input = getConformingInput();
Chris@350 58 if (!input) {
Chris@361 59 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@350 60 return;
Chris@350 61 }
Chris@320 62
Chris@350 63 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 64 if (!m_plugin) {
Chris@361 65 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@320 66 return;
Chris@320 67 }
Chris@320 68
Chris@350 69 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@350 70 (m_transform, m_plugin);
Chris@343 71
Chris@350 72 TransformFactory::getInstance()->setPluginParameters
Chris@350 73 (m_transform, m_plugin);
Chris@320 74
Chris@320 75 size_t channelCount = input->getChannelCount();
Chris@320 76 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 77 channelCount = 1;
Chris@320 78 }
Chris@320 79 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@361 80 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 81 .arg(pluginId)
Chris@361 82 .arg(m_plugin->getMinChannelCount())
Chris@361 83 .arg(m_plugin->getMaxChannelCount())
Chris@361 84 .arg(input->getChannelCount());
Chris@320 85 return;
Chris@320 86 }
Chris@320 87
Chris@320 88 std::cerr << "Initialising feature extraction plugin with channels = "
Chris@350 89 << channelCount << ", step = " << m_transform.getStepSize()
Chris@350 90 << ", block = " << m_transform.getBlockSize() << std::endl;
Chris@320 91
Chris@320 92 if (!m_plugin->initialise(channelCount,
Chris@350 93 m_transform.getStepSize(),
Chris@350 94 m_transform.getBlockSize())) {
Chris@361 95
Chris@361 96 size_t pstep = m_transform.getStepSize();
Chris@361 97 size_t pblock = m_transform.getBlockSize();
Chris@361 98
Chris@361 99 m_transform.setStepSize(0);
Chris@361 100 m_transform.setBlockSize(0);
Chris@361 101 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@361 102 (m_transform, m_plugin);
Chris@361 103
Chris@361 104 if (m_transform.getStepSize() != pstep ||
Chris@361 105 m_transform.getBlockSize() != pblock) {
Chris@361 106
Chris@361 107 if (!m_plugin->initialise(channelCount,
Chris@361 108 m_transform.getStepSize(),
Chris@361 109 m_transform.getBlockSize())) {
Chris@361 110
Chris@361 111 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 112 return;
Chris@361 113
Chris@361 114 } else {
Chris@361 115
Chris@361 116 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 117 .arg(pluginId)
Chris@361 118 .arg(pstep)
Chris@361 119 .arg(pblock)
Chris@361 120 .arg(m_transform.getStepSize())
Chris@361 121 .arg(m_transform.getBlockSize());
Chris@361 122 }
Chris@361 123
Chris@361 124 } else {
Chris@361 125
Chris@361 126 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 127 return;
Chris@361 128 }
Chris@320 129 }
Chris@320 130
Chris@366 131 if (m_transform.getPluginVersion() != "") {
Chris@366 132 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@366 133 if (pv != m_transform.getPluginVersion()) {
Chris@366 134 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@366 135 .arg(m_transform.getPluginVersion())
Chris@366 136 .arg(pluginId)
Chris@366 137 .arg(pv);
Chris@366 138 if (m_message != "") {
Chris@366 139 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 140 } else {
Chris@366 141 m_message = vm;
Chris@366 142 }
Chris@366 143 }
Chris@366 144 }
Chris@366 145
Chris@320 146 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 147
Chris@320 148 if (outputs.empty()) {
Chris@361 149 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@320 150 return;
Chris@320 151 }
Chris@320 152
Chris@320 153 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@429 154 // std::cerr << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput().toStdString() << "\"" << std::endl;
Chris@350 155 if (m_transform.getOutput() == "" ||
Chris@350 156 outputs[i].identifier == m_transform.getOutput().toStdString()) {
Chris@320 157 m_outputFeatureNo = i;
Chris@441 158 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
Chris@320 159 break;
Chris@320 160 }
Chris@320 161 }
Chris@320 162
Chris@320 163 if (!m_descriptor) {
Chris@361 164 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@361 165 .arg(pluginId)
Chris@361 166 .arg(m_transform.getOutput());
Chris@320 167 return;
Chris@320 168 }
Chris@320 169
Chris@331 170 // std::cerr << "FeatureExtractionModelTransformer: output sample type "
Chris@320 171 // << m_descriptor->sampleType << std::endl;
Chris@320 172
Chris@320 173 int binCount = 1;
Chris@320 174 float minValue = 0.0, maxValue = 0.0;
Chris@320 175 bool haveExtents = false;
Chris@320 176
Chris@320 177 if (m_descriptor->hasFixedBinCount) {
Chris@320 178 binCount = m_descriptor->binCount;
Chris@320 179 }
Chris@320 180
Chris@331 181 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@320 182 // << binCount << std::endl;
Chris@320 183
Chris@320 184 if (binCount > 0 && m_descriptor->hasKnownExtents) {
Chris@320 185 minValue = m_descriptor->minValue;
Chris@320 186 maxValue = m_descriptor->maxValue;
Chris@320 187 haveExtents = true;
Chris@320 188 }
Chris@320 189
Chris@350 190 size_t modelRate = input->getSampleRate();
Chris@320 191 size_t modelResolution = 1;
Chris@320 192
Chris@320 193 switch (m_descriptor->sampleType) {
Chris@320 194
Chris@320 195 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@320 196 if (m_descriptor->sampleRate != 0.0) {
Chris@320 197 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@320 198 }
Chris@320 199 break;
Chris@320 200
Chris@320 201 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@350 202 modelResolution = m_transform.getStepSize();
Chris@320 203 break;
Chris@320 204
Chris@320 205 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@451 206 //!!! SV doesn't actually support display of models that have
Chris@451 207 //!!! different underlying rates together -- so we always set
Chris@451 208 //!!! the model rate to be the input model's rate, and adjust
Chris@451 209 //!!! the resolution appropriately. We can't properly display
Chris@451 210 //!!! data with a higher resolution than the base model at all
Chris@451 211 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
Chris@451 212 if (m_descriptor->sampleRate > input->getSampleRate()) {
Chris@451 213 modelResolution = 1;
Chris@451 214 } else {
Chris@451 215 modelResolution = size_t(input->getSampleRate() /
Chris@451 216 m_descriptor->sampleRate);
Chris@451 217 }
Chris@320 218 break;
Chris@320 219 }
Chris@320 220
Chris@441 221 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 222
Chris@441 223 if (binCount == 0 &&
Chris@441 224 (preDurationPlugin || !m_descriptor->hasDuration)) {
Chris@320 225
Chris@445 226 // Anything with no value and no duration is an instant
Chris@445 227
Chris@320 228 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
Chris@320 229 false);
Chris@320 230
Chris@441 231 } else if ((preDurationPlugin && binCount > 1 &&
Chris@441 232 (m_descriptor->sampleType ==
Chris@441 233 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@441 234 (!preDurationPlugin && m_descriptor->hasDuration)) {
Chris@441 235
Chris@441 236 // For plugins using the old v1 API without explicit duration,
Chris@441 237 // we treat anything that has multiple bins (i.e. that has the
Chris@441 238 // potential to have value and duration) and a variable sample
Chris@441 239 // rate as a note model, taking its values as pitch, duration
Chris@441 240 // and velocity (if present) respectively. This is the same
Chris@441 241 // behaviour as always applied by SV to these plugins in the
Chris@441 242 // past.
Chris@441 243
Chris@441 244 // For plugins with the newer API, we treat anything with
Chris@441 245 // duration as either a note model with pitch and velocity, or
Chris@441 246 // a region model.
Chris@441 247
Chris@441 248 // How do we know whether it's an interval or note model?
Chris@441 249 // What's the essential difference? Is a note model any
Chris@441 250 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 251 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 252 // note model always have velocity? This is a good question
Chris@441 253 // to be addressed by accompanying RDF, but for the moment we
Chris@441 254 // will do the following...
Chris@441 255
Chris@441 256 bool isNoteModel = false;
Chris@441 257
Chris@441 258 // Regions have only value (and duration -- we can't extract a
Chris@441 259 // region model from an old-style plugin that doesn't support
Chris@441 260 // duration)
Chris@441 261 if (binCount > 1) isNoteModel = true;
Chris@441 262
Chris@441 263 // Regions do not have units of Hz (a sweeping assumption!)
Chris@441 264 if (m_descriptor->unit == "Hz") isNoteModel = true;
Chris@441 265
Chris@441 266 // If we had a "sparse 3D model", we would have the additional
Chris@441 267 // problem of determining whether to use that here (if bin
Chris@441 268 // count > 1). But we don't.
Chris@441 269
Chris@441 270 if (isNoteModel) {
Chris@441 271
Chris@441 272 NoteModel *model;
Chris@441 273 if (haveExtents) {
Chris@441 274 model = new NoteModel
Chris@441 275 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 276 } else {
Chris@441 277 model = new NoteModel
Chris@441 278 (modelRate, modelResolution, false);
Chris@441 279 }
Chris@441 280 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 281 m_output = model;
Chris@441 282
Chris@441 283 } else {
Chris@441 284
Chris@441 285 RegionModel *model;
Chris@441 286 if (haveExtents) {
Chris@441 287 model = new RegionModel
Chris@441 288 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 289 } else {
Chris@441 290 model = new RegionModel
Chris@441 291 (modelRate, modelResolution, false);
Chris@441 292 }
Chris@441 293 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 294 m_output = model;
Chris@441 295 }
Chris@441 296
Chris@441 297 } else if (binCount == 1 ||
Chris@441 298 (m_descriptor->sampleType ==
Chris@441 299 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 300
Chris@441 301 // Anything that is not a 1D, note, or interval model and that
Chris@441 302 // has only one value per result must be a sparse time value
Chris@441 303 // model.
Chris@441 304
Chris@441 305 // Anything that is not a 1D, note, or interval model and that
Chris@441 306 // has a variable sample rate is also treated as a sparse time
Chris@441 307 // value model regardless of its bin count, because we lack a
Chris@441 308 // sparse 3D model.
Chris@320 309
Chris@320 310 SparseTimeValueModel *model;
Chris@320 311 if (haveExtents) {
Chris@320 312 model = new SparseTimeValueModel
Chris@320 313 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 314 } else {
Chris@320 315 model = new SparseTimeValueModel
Chris@320 316 (modelRate, modelResolution, false);
Chris@320 317 }
Chris@320 318 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
Chris@320 319
Chris@320 320 m_output = model;
Chris@320 321
Chris@441 322 } else {
Chris@320 323
Chris@441 324 // Anything that is not a 1D, note, or interval model and that
Chris@441 325 // has a fixed sample rate and more than one value per result
Chris@441 326 // must be a dense 3D model.
Chris@320 327
Chris@320 328 EditableDenseThreeDimensionalModel *model =
Chris@320 329 new EditableDenseThreeDimensionalModel
Chris@535 330 (modelRate, modelResolution, binCount,
Chris@535 331 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 332 false);
Chris@320 333
Chris@320 334 if (!m_descriptor->binNames.empty()) {
Chris@320 335 std::vector<QString> names;
Chris@320 336 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
Chris@320 337 names.push_back(m_descriptor->binNames[i].c_str());
Chris@320 338 }
Chris@320 339 model->setBinNames(names);
Chris@320 340 }
Chris@320 341
Chris@320 342 m_output = model;
Chris@320 343 }
Chris@333 344
Chris@350 345 if (m_output) m_output->setSourceModel(input);
Chris@320 346 }
Chris@320 347
Chris@331 348 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 349 {
Chris@436 350 // std::cerr << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << std::endl;
Chris@320 351 delete m_plugin;
Chris@320 352 delete m_descriptor;
Chris@320 353 }
Chris@320 354
Chris@320 355 DenseTimeValueModel *
Chris@350 356 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 357 {
Chris@408 358 // std::cerr << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << std::endl;
Chris@408 359
Chris@320 360 DenseTimeValueModel *dtvm =
Chris@320 361 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 362 if (!dtvm) {
Chris@350 363 std::cerr << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
Chris@320 364 }
Chris@320 365 return dtvm;
Chris@320 366 }
Chris@320 367
Chris@320 368 void
Chris@331 369 FeatureExtractionModelTransformer::run()
Chris@320 370 {
Chris@350 371 DenseTimeValueModel *input = getConformingInput();
Chris@320 372 if (!input) return;
Chris@320 373
Chris@320 374 if (!m_output) return;
Chris@320 375
Chris@497 376 while (!input->isReady() && !m_abandoned) {
Chris@331 377 std::cerr << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << std::endl;
Chris@497 378 usleep(500000);
Chris@320 379 }
Chris@497 380 if (m_abandoned) return;
Chris@320 381
Chris@350 382 size_t sampleRate = input->getSampleRate();
Chris@320 383
Chris@320 384 size_t channelCount = input->getChannelCount();
Chris@320 385 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 386 channelCount = 1;
Chris@320 387 }
Chris@320 388
Chris@320 389 float **buffers = new float*[channelCount];
Chris@320 390 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 391 buffers[ch] = new float[m_transform.getBlockSize() + 2];
Chris@320 392 }
Chris@320 393
Chris@350 394 size_t stepSize = m_transform.getStepSize();
Chris@350 395 size_t blockSize = m_transform.getBlockSize();
Chris@350 396
Chris@320 397 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 398 Vamp::Plugin::FrequencyDomain);
Chris@320 399 std::vector<FFTModel *> fftModels;
Chris@320 400
Chris@320 401 if (frequencyDomain) {
Chris@320 402 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 403 FFTModel *model = new FFTModel
Chris@350 404 (getConformingInput(),
Chris@350 405 channelCount == 1 ? m_input.getChannel() : ch,
Chris@350 406 m_transform.getWindowType(),
Chris@350 407 blockSize,
Chris@350 408 stepSize,
Chris@350 409 blockSize,
Chris@334 410 false,
Chris@334 411 StorageAdviser::PrecisionCritical);
Chris@320 412 if (!model->isOK()) {
Chris@320 413 delete model;
Chris@320 414 setCompletion(100);
Chris@387 415 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@387 416 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
Chris@320 417 }
Chris@320 418 model->resume();
Chris@320 419 fftModels.push_back(model);
Chris@320 420 }
Chris@320 421 }
Chris@320 422
Chris@350 423 long startFrame = m_input.getModel()->getStartFrame();
Chris@350 424 long endFrame = m_input.getModel()->getEndFrame();
Chris@320 425
Chris@350 426 RealTime contextStartRT = m_transform.getStartTime();
Chris@350 427 RealTime contextDurationRT = m_transform.getDuration();
Chris@350 428
Chris@350 429 long contextStart =
Chris@350 430 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 431
Chris@350 432 long contextDuration =
Chris@350 433 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 434
Chris@320 435 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 436 contextStart = startFrame;
Chris@320 437 }
Chris@320 438
Chris@320 439 if (contextDuration == 0) {
Chris@320 440 contextDuration = endFrame - contextStart;
Chris@320 441 }
Chris@320 442 if (contextStart + contextDuration > endFrame) {
Chris@320 443 contextDuration = endFrame - contextStart;
Chris@320 444 }
Chris@320 445
Chris@320 446 long blockFrame = contextStart;
Chris@320 447
Chris@320 448 long prevCompletion = 0;
Chris@320 449
Chris@320 450 setCompletion(0);
Chris@320 451
Chris@556 452 float *reals = 0;
Chris@556 453 float *imaginaries = 0;
Chris@556 454 if (frequencyDomain) {
Chris@556 455 reals = new float[blockSize/2 + 1];
Chris@556 456 imaginaries = new float[blockSize/2 + 1];
Chris@556 457 }
Chris@556 458
Chris@320 459 while (!m_abandoned) {
Chris@320 460
Chris@320 461 if (frequencyDomain) {
Chris@350 462 if (blockFrame - int(blockSize)/2 >
Chris@320 463 contextStart + contextDuration) break;
Chris@320 464 } else {
Chris@320 465 if (blockFrame >=
Chris@320 466 contextStart + contextDuration) break;
Chris@320 467 }
Chris@320 468
Chris@331 469 // std::cerr << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@320 470 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@350 471 // << blockSize << std::endl;
Chris@320 472
Chris@320 473 long completion =
Chris@350 474 (((blockFrame - contextStart) / stepSize) * 99) /
Chris@557 475 (contextDuration / stepSize + 1);
Chris@320 476
Chris@350 477 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 478
Chris@363 479 if (frequencyDomain) {
Chris@363 480 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 481 int column = (blockFrame - startFrame) / stepSize;
Chris@556 482 fftModels[ch]->getValuesAt(column, reals, imaginaries);
Chris@350 483 for (size_t i = 0; i <= blockSize/2; ++i) {
Chris@556 484 buffers[ch][i*2] = reals[i];
Chris@556 485 buffers[ch][i*2+1] = imaginaries[i];
Chris@320 486 }
Chris@363 487 }
Chris@363 488 } else {
Chris@363 489 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@320 490 }
Chris@320 491
Chris@497 492 if (m_abandoned) break;
Chris@497 493
Chris@320 494 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@320 495 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@320 496
Chris@497 497 if (m_abandoned) break;
Chris@497 498
Chris@320 499 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@320 500 Vamp::Plugin::Feature feature =
Chris@320 501 features[m_outputFeatureNo][fi];
Chris@320 502 addFeature(blockFrame, feature);
Chris@320 503 }
Chris@320 504
Chris@320 505 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@320 506 setCompletion(completion);
Chris@320 507 prevCompletion = completion;
Chris@320 508 }
Chris@320 509
Chris@350 510 blockFrame += stepSize;
Chris@320 511 }
Chris@320 512
Chris@497 513 if (!m_abandoned) {
Chris@497 514 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@320 515
Chris@497 516 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@497 517 Vamp::Plugin::Feature feature =
Chris@497 518 features[m_outputFeatureNo][fi];
Chris@497 519 addFeature(blockFrame, feature);
Chris@497 520 }
Chris@497 521 }
Chris@320 522
Chris@497 523 setCompletion(100);
Chris@320 524
Chris@320 525 if (frequencyDomain) {
Chris@320 526 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 527 delete fftModels[ch];
Chris@320 528 }
Chris@556 529 delete[] reals;
Chris@556 530 delete[] imaginaries;
Chris@320 531 }
Chris@320 532 }
Chris@320 533
Chris@320 534 void
Chris@363 535 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@363 536 long startFrame, long size,
Chris@363 537 float **buffers)
Chris@320 538 {
Chris@320 539 long offset = 0;
Chris@320 540
Chris@320 541 if (startFrame < 0) {
Chris@363 542 for (int c = 0; c < channelCount; ++c) {
Chris@363 543 for (int i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 544 buffers[c][i] = 0.0f;
Chris@363 545 }
Chris@320 546 }
Chris@320 547 offset = -startFrame;
Chris@320 548 size -= offset;
Chris@320 549 if (size <= 0) return;
Chris@320 550 startFrame = 0;
Chris@320 551 }
Chris@320 552
Chris@350 553 DenseTimeValueModel *input = getConformingInput();
Chris@350 554 if (!input) return;
Chris@363 555
Chris@363 556 long got = 0;
Chris@350 557
Chris@363 558 if (channelCount == 1) {
Chris@363 559
Chris@363 560 got = input->getData(m_input.getChannel(), startFrame, size,
Chris@363 561 buffers[0] + offset);
Chris@363 562
Chris@363 563 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 564 // use mean instead of sum, as plugin input
Chris@363 565 float cc = float(input->getChannelCount());
Chris@363 566 for (long i = 0; i < size; ++i) {
Chris@363 567 buffers[0][i + offset] /= cc;
Chris@363 568 }
Chris@363 569 }
Chris@363 570
Chris@363 571 } else {
Chris@363 572
Chris@363 573 float **writebuf = buffers;
Chris@363 574 if (offset > 0) {
Chris@363 575 writebuf = new float *[channelCount];
Chris@363 576 for (int i = 0; i < channelCount; ++i) {
Chris@363 577 writebuf[i] = buffers[i] + offset;
Chris@363 578 }
Chris@363 579 }
Chris@363 580
Chris@363 581 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
Chris@363 582
Chris@363 583 if (writebuf != buffers) delete[] writebuf;
Chris@363 584 }
Chris@320 585
Chris@320 586 while (got < size) {
Chris@363 587 for (int c = 0; c < channelCount; ++c) {
Chris@363 588 buffers[c][got + offset] = 0.0;
Chris@363 589 }
Chris@320 590 ++got;
Chris@320 591 }
Chris@320 592 }
Chris@320 593
Chris@320 594 void
Chris@331 595 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
Chris@320 596 const Vamp::Plugin::Feature &feature)
Chris@320 597 {
Chris@350 598 size_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 599
Chris@331 600 // std::cerr << "FeatureExtractionModelTransformer::addFeature("
Chris@320 601 // << blockFrame << ")" << std::endl;
Chris@320 602
Chris@320 603 int binCount = 1;
Chris@320 604 if (m_descriptor->hasFixedBinCount) {
Chris@320 605 binCount = m_descriptor->binCount;
Chris@320 606 }
Chris@320 607
Chris@320 608 size_t frame = blockFrame;
Chris@320 609
Chris@320 610 if (m_descriptor->sampleType ==
Chris@320 611 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 612
Chris@320 613 if (!feature.hasTimestamp) {
Chris@320 614 std::cerr
Chris@331 615 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@320 616 << "Feature has variable sample rate but no timestamp!"
Chris@320 617 << std::endl;
Chris@320 618 return;
Chris@320 619 } else {
Chris@320 620 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@320 621 }
Chris@320 622
Chris@320 623 } else if (m_descriptor->sampleType ==
Chris@320 624 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 625
Chris@320 626 if (feature.hasTimestamp) {
Chris@320 627 //!!! warning: sampleRate may be non-integral
Chris@320 628 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
Chris@451 629 //!!! see comment above when setting up modelResolution and modelRate
Chris@451 630 // lrintf(m_descriptor->sampleRate));
Chris@451 631 inputRate);
Chris@320 632 } else {
Chris@320 633 frame = m_output->getEndFrame();
Chris@320 634 }
Chris@320 635 }
Chris@320 636
Chris@441 637 // Rather than repeat the complicated tests from the constructor
Chris@441 638 // to determine what sort of model we must be adding the features
Chris@441 639 // to, we instead test what sort of model the constructor decided
Chris@441 640 // to create.
Chris@320 641
Chris@441 642 if (isOutput<SparseOneDimensionalModel>()) {
Chris@441 643
Chris@441 644 SparseOneDimensionalModel *model =
Chris@350 645 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 646 if (!model) return;
Chris@350 647
Chris@441 648 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 649 (frame, feature.label.c_str()));
Chris@320 650
Chris@441 651 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 652
Chris@350 653 SparseTimeValueModel *model =
Chris@350 654 getConformingOutput<SparseTimeValueModel>();
Chris@320 655 if (!model) return;
Chris@350 656
Chris@454 657 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 658
Chris@454 659 float value = feature.values[i];
Chris@454 660
Chris@454 661 QString label = feature.label.c_str();
Chris@454 662 if (feature.values.size() > 1) {
Chris@454 663 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 664 }
Chris@454 665
Chris@454 666 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
Chris@454 667 }
Chris@320 668
Chris@441 669 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
Chris@320 670
Chris@441 671 int index = 0;
Chris@441 672
Chris@441 673 float value = 0.0;
Chris@441 674 if (feature.values.size() > index) {
Chris@441 675 value = feature.values[index++];
Chris@441 676 }
Chris@320 677
Chris@320 678 float duration = 1;
Chris@441 679 if (feature.hasDuration) {
Chris@441 680 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 681 } else {
Chris@441 682 if (feature.values.size() > index) {
Chris@441 683 duration = feature.values[index++];
Chris@441 684 }
Chris@441 685 }
Chris@320 686
Chris@441 687 if (isOutput<NoteModel>()) {
Chris@320 688
Chris@441 689 float velocity = 100;
Chris@441 690 if (feature.values.size() > index) {
Chris@441 691 velocity = feature.values[index++];
Chris@441 692 }
Chris@441 693 if (velocity < 0) velocity = 127;
Chris@441 694 if (velocity > 127) velocity = 127;
Chris@320 695
Chris@441 696 NoteModel *model = getConformingOutput<NoteModel>();
Chris@441 697 if (!model) return;
Chris@441 698 model->addPoint(NoteModel::Point(frame, value, // value is pitch
Chris@441 699 lrintf(duration),
Chris@441 700 velocity / 127.f,
Chris@441 701 feature.label.c_str()));
Chris@441 702 } else {
Chris@441 703 RegionModel *model = getConformingOutput<RegionModel>();
Chris@454 704 if (!model) return;
Chris@454 705
Chris@474 706 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 707
Chris@454 708 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 709
Chris@454 710 float value = feature.values[i];
Chris@454 711
Chris@454 712 QString label = feature.label.c_str();
Chris@454 713 if (feature.values.size() > 1) {
Chris@454 714 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 715 }
Chris@454 716
Chris@454 717 model->addPoint(RegionModel::Point(frame, value,
Chris@454 718 lrintf(duration),
Chris@454 719 label));
Chris@454 720 }
Chris@454 721 } else {
Chris@454 722
Chris@441 723 model->addPoint(RegionModel::Point(frame, value,
Chris@441 724 lrintf(duration),
Chris@441 725 feature.label.c_str()));
Chris@454 726 }
Chris@441 727 }
Chris@320 728
Chris@441 729 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 730
Chris@533 731 DenseThreeDimensionalModel::Column values =
Chris@533 732 DenseThreeDimensionalModel::Column::fromStdVector(feature.values);
Chris@320 733
Chris@320 734 EditableDenseThreeDimensionalModel *model =
Chris@350 735 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 736 if (!model) return;
Chris@320 737
Chris@320 738 model->setColumn(frame / model->getResolution(), values);
Chris@441 739
Chris@441 740 } else {
Chris@441 741 std::cerr << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << std::endl;
Chris@320 742 }
Chris@320 743 }
Chris@320 744
Chris@320 745 void
Chris@331 746 FeatureExtractionModelTransformer::setCompletion(int completion)
Chris@320 747 {
Chris@320 748 int binCount = 1;
Chris@320 749 if (m_descriptor->hasFixedBinCount) {
Chris@320 750 binCount = m_descriptor->binCount;
Chris@320 751 }
Chris@320 752
Chris@331 753 // std::cerr << "FeatureExtractionModelTransformer::setCompletion("
Chris@320 754 // << completion << ")" << std::endl;
Chris@320 755
Chris@441 756 if (isOutput<SparseOneDimensionalModel>()) {
Chris@320 757
Chris@350 758 SparseOneDimensionalModel *model =
Chris@350 759 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 760 if (!model) return;
Chris@441 761 model->setCompletion(completion, true);
Chris@320 762
Chris@441 763 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 764
Chris@350 765 SparseTimeValueModel *model =
Chris@350 766 getConformingOutput<SparseTimeValueModel>();
Chris@320 767 if (!model) return;
Chris@441 768 model->setCompletion(completion, true);
Chris@320 769
Chris@441 770 } else if (isOutput<NoteModel>()) {
Chris@320 771
Chris@441 772 NoteModel *model = getConformingOutput<NoteModel>();
Chris@320 773 if (!model) return;
Chris@441 774 model->setCompletion(completion, true);
Chris@320 775
Chris@441 776 } else if (isOutput<RegionModel>()) {
Chris@441 777
Chris@441 778 RegionModel *model = getConformingOutput<RegionModel>();
Chris@441 779 if (!model) return;
Chris@441 780 model->setCompletion(completion, true);
Chris@441 781
Chris@441 782 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 783
Chris@320 784 EditableDenseThreeDimensionalModel *model =
Chris@350 785 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 786 if (!model) return;
Chris@350 787 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 788 }
Chris@320 789 }
Chris@320 790