annotate transform/FeatureExtractionModelTransformer.cpp @ 497:b6dc6c7f402c

Various fixes: * Fix handling of HTTP redirects (avoiding crashes... I hope) * Fix failure to delete FFT models when a feature extraction model transformer was abandoned (also a cause of crashes in the past) * Fix deadlock when said transform was abandoned before its source model was ready because the session was being cleared (and so the source model would never be ready)
author Chris Cannam
date Fri, 28 Nov 2008 13:36:13 +0000
parents a70dcfed59c1
children 55ad231c9db7
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@320 19 #include "plugin/PluginXml.h"
Chris@475 20 #include <vamp-hostsdk/Plugin.h>
Chris@320 21
Chris@320 22 #include "data/model/Model.h"
Chris@320 23 #include "base/Window.h"
Chris@387 24 #include "base/Exceptions.h"
Chris@320 25 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 26 #include "data/model/SparseTimeValueModel.h"
Chris@320 27 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 28 #include "data/model/DenseTimeValueModel.h"
Chris@320 29 #include "data/model/NoteModel.h"
Chris@441 30 #include "data/model/RegionModel.h"
Chris@320 31 #include "data/model/FFTModel.h"
Chris@320 32 #include "data/model/WaveFileModel.h"
Chris@320 33
Chris@350 34 #include "TransformFactory.h"
Chris@350 35
Chris@320 36 #include <iostream>
Chris@320 37
Chris@350 38 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@350 39 const Transform &transform) :
Chris@350 40 ModelTransformer(in, transform),
Chris@320 41 m_plugin(0),
Chris@320 42 m_descriptor(0),
Chris@320 43 m_outputFeatureNo(0)
Chris@320 44 {
Chris@350 45 // std::cerr << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId.toStdString() << ", outputName " << m_transform.getOutput().toStdString() << std::endl;
Chris@350 46
Chris@350 47 QString pluginId = transform.getPluginIdentifier();
Chris@320 48
Chris@320 49 FeatureExtractionPluginFactory *factory =
Chris@320 50 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@320 51
Chris@320 52 if (!factory) {
Chris@361 53 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@320 54 return;
Chris@320 55 }
Chris@320 56
Chris@350 57 DenseTimeValueModel *input = getConformingInput();
Chris@350 58 if (!input) {
Chris@361 59 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@350 60 return;
Chris@350 61 }
Chris@320 62
Chris@350 63 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 64 if (!m_plugin) {
Chris@361 65 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@320 66 return;
Chris@320 67 }
Chris@320 68
Chris@350 69 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@350 70 (m_transform, m_plugin);
Chris@343 71
Chris@350 72 TransformFactory::getInstance()->setPluginParameters
Chris@350 73 (m_transform, m_plugin);
Chris@320 74
Chris@320 75 size_t channelCount = input->getChannelCount();
Chris@320 76 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 77 channelCount = 1;
Chris@320 78 }
Chris@320 79 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@361 80 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 81 .arg(pluginId)
Chris@361 82 .arg(m_plugin->getMinChannelCount())
Chris@361 83 .arg(m_plugin->getMaxChannelCount())
Chris@361 84 .arg(input->getChannelCount());
Chris@320 85 return;
Chris@320 86 }
Chris@320 87
Chris@320 88 std::cerr << "Initialising feature extraction plugin with channels = "
Chris@350 89 << channelCount << ", step = " << m_transform.getStepSize()
Chris@350 90 << ", block = " << m_transform.getBlockSize() << std::endl;
Chris@320 91
Chris@320 92 if (!m_plugin->initialise(channelCount,
Chris@350 93 m_transform.getStepSize(),
Chris@350 94 m_transform.getBlockSize())) {
Chris@361 95
Chris@361 96 size_t pstep = m_transform.getStepSize();
Chris@361 97 size_t pblock = m_transform.getBlockSize();
Chris@361 98
Chris@361 99 m_transform.setStepSize(0);
Chris@361 100 m_transform.setBlockSize(0);
Chris@361 101 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@361 102 (m_transform, m_plugin);
Chris@361 103
Chris@361 104 if (m_transform.getStepSize() != pstep ||
Chris@361 105 m_transform.getBlockSize() != pblock) {
Chris@361 106
Chris@361 107 if (!m_plugin->initialise(channelCount,
Chris@361 108 m_transform.getStepSize(),
Chris@361 109 m_transform.getBlockSize())) {
Chris@361 110
Chris@361 111 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 112 return;
Chris@361 113
Chris@361 114 } else {
Chris@361 115
Chris@361 116 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 117 .arg(pluginId)
Chris@361 118 .arg(pstep)
Chris@361 119 .arg(pblock)
Chris@361 120 .arg(m_transform.getStepSize())
Chris@361 121 .arg(m_transform.getBlockSize());
Chris@361 122 }
Chris@361 123
Chris@361 124 } else {
Chris@361 125
Chris@361 126 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 127 return;
Chris@361 128 }
Chris@320 129 }
Chris@320 130
Chris@366 131 if (m_transform.getPluginVersion() != "") {
Chris@366 132 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@366 133 if (pv != m_transform.getPluginVersion()) {
Chris@366 134 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@366 135 .arg(m_transform.getPluginVersion())
Chris@366 136 .arg(pluginId)
Chris@366 137 .arg(pv);
Chris@366 138 if (m_message != "") {
Chris@366 139 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 140 } else {
Chris@366 141 m_message = vm;
Chris@366 142 }
Chris@366 143 }
Chris@366 144 }
Chris@366 145
Chris@320 146 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 147
Chris@320 148 if (outputs.empty()) {
Chris@361 149 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@320 150 return;
Chris@320 151 }
Chris@320 152
Chris@320 153 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@429 154 // std::cerr << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput().toStdString() << "\"" << std::endl;
Chris@350 155 if (m_transform.getOutput() == "" ||
Chris@350 156 outputs[i].identifier == m_transform.getOutput().toStdString()) {
Chris@320 157 m_outputFeatureNo = i;
Chris@441 158 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
Chris@320 159 break;
Chris@320 160 }
Chris@320 161 }
Chris@320 162
Chris@320 163 if (!m_descriptor) {
Chris@361 164 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@361 165 .arg(pluginId)
Chris@361 166 .arg(m_transform.getOutput());
Chris@320 167 return;
Chris@320 168 }
Chris@320 169
Chris@331 170 // std::cerr << "FeatureExtractionModelTransformer: output sample type "
Chris@320 171 // << m_descriptor->sampleType << std::endl;
Chris@320 172
Chris@320 173 int binCount = 1;
Chris@320 174 float minValue = 0.0, maxValue = 0.0;
Chris@320 175 bool haveExtents = false;
Chris@320 176
Chris@320 177 if (m_descriptor->hasFixedBinCount) {
Chris@320 178 binCount = m_descriptor->binCount;
Chris@320 179 }
Chris@320 180
Chris@331 181 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@320 182 // << binCount << std::endl;
Chris@320 183
Chris@320 184 if (binCount > 0 && m_descriptor->hasKnownExtents) {
Chris@320 185 minValue = m_descriptor->minValue;
Chris@320 186 maxValue = m_descriptor->maxValue;
Chris@320 187 haveExtents = true;
Chris@320 188 }
Chris@320 189
Chris@350 190 size_t modelRate = input->getSampleRate();
Chris@320 191 size_t modelResolution = 1;
Chris@320 192
Chris@320 193 switch (m_descriptor->sampleType) {
Chris@320 194
Chris@320 195 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@320 196 if (m_descriptor->sampleRate != 0.0) {
Chris@320 197 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@320 198 }
Chris@320 199 break;
Chris@320 200
Chris@320 201 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@350 202 modelResolution = m_transform.getStepSize();
Chris@320 203 break;
Chris@320 204
Chris@320 205 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@451 206 //!!! SV doesn't actually support display of models that have
Chris@451 207 //!!! different underlying rates together -- so we always set
Chris@451 208 //!!! the model rate to be the input model's rate, and adjust
Chris@451 209 //!!! the resolution appropriately. We can't properly display
Chris@451 210 //!!! data with a higher resolution than the base model at all
Chris@451 211 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
Chris@451 212 if (m_descriptor->sampleRate > input->getSampleRate()) {
Chris@451 213 modelResolution = 1;
Chris@451 214 } else {
Chris@451 215 modelResolution = size_t(input->getSampleRate() /
Chris@451 216 m_descriptor->sampleRate);
Chris@451 217 }
Chris@320 218 break;
Chris@320 219 }
Chris@320 220
Chris@441 221 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 222
Chris@441 223 if (binCount == 0 &&
Chris@441 224 (preDurationPlugin || !m_descriptor->hasDuration)) {
Chris@320 225
Chris@445 226 // Anything with no value and no duration is an instant
Chris@445 227
Chris@320 228 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
Chris@320 229 false);
Chris@320 230
Chris@441 231 } else if ((preDurationPlugin && binCount > 1 &&
Chris@441 232 (m_descriptor->sampleType ==
Chris@441 233 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@441 234 (!preDurationPlugin && m_descriptor->hasDuration)) {
Chris@441 235
Chris@441 236 // For plugins using the old v1 API without explicit duration,
Chris@441 237 // we treat anything that has multiple bins (i.e. that has the
Chris@441 238 // potential to have value and duration) and a variable sample
Chris@441 239 // rate as a note model, taking its values as pitch, duration
Chris@441 240 // and velocity (if present) respectively. This is the same
Chris@441 241 // behaviour as always applied by SV to these plugins in the
Chris@441 242 // past.
Chris@441 243
Chris@441 244 // For plugins with the newer API, we treat anything with
Chris@441 245 // duration as either a note model with pitch and velocity, or
Chris@441 246 // a region model.
Chris@441 247
Chris@441 248 // How do we know whether it's an interval or note model?
Chris@441 249 // What's the essential difference? Is a note model any
Chris@441 250 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 251 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 252 // note model always have velocity? This is a good question
Chris@441 253 // to be addressed by accompanying RDF, but for the moment we
Chris@441 254 // will do the following...
Chris@441 255
Chris@441 256 bool isNoteModel = false;
Chris@441 257
Chris@441 258 // Regions have only value (and duration -- we can't extract a
Chris@441 259 // region model from an old-style plugin that doesn't support
Chris@441 260 // duration)
Chris@441 261 if (binCount > 1) isNoteModel = true;
Chris@441 262
Chris@441 263 // Regions do not have units of Hz (a sweeping assumption!)
Chris@441 264 if (m_descriptor->unit == "Hz") isNoteModel = true;
Chris@441 265
Chris@441 266 // If we had a "sparse 3D model", we would have the additional
Chris@441 267 // problem of determining whether to use that here (if bin
Chris@441 268 // count > 1). But we don't.
Chris@441 269
Chris@441 270 if (isNoteModel) {
Chris@441 271
Chris@441 272 NoteModel *model;
Chris@441 273 if (haveExtents) {
Chris@441 274 model = new NoteModel
Chris@441 275 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 276 } else {
Chris@441 277 model = new NoteModel
Chris@441 278 (modelRate, modelResolution, false);
Chris@441 279 }
Chris@441 280 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 281 m_output = model;
Chris@441 282
Chris@441 283 } else {
Chris@441 284
Chris@441 285 RegionModel *model;
Chris@441 286 if (haveExtents) {
Chris@441 287 model = new RegionModel
Chris@441 288 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 289 } else {
Chris@441 290 model = new RegionModel
Chris@441 291 (modelRate, modelResolution, false);
Chris@441 292 }
Chris@441 293 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 294 m_output = model;
Chris@441 295 }
Chris@441 296
Chris@441 297 } else if (binCount == 1 ||
Chris@441 298 (m_descriptor->sampleType ==
Chris@441 299 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 300
Chris@441 301 // Anything that is not a 1D, note, or interval model and that
Chris@441 302 // has only one value per result must be a sparse time value
Chris@441 303 // model.
Chris@441 304
Chris@441 305 // Anything that is not a 1D, note, or interval model and that
Chris@441 306 // has a variable sample rate is also treated as a sparse time
Chris@441 307 // value model regardless of its bin count, because we lack a
Chris@441 308 // sparse 3D model.
Chris@320 309
Chris@320 310 SparseTimeValueModel *model;
Chris@320 311 if (haveExtents) {
Chris@320 312 model = new SparseTimeValueModel
Chris@320 313 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 314 } else {
Chris@320 315 model = new SparseTimeValueModel
Chris@320 316 (modelRate, modelResolution, false);
Chris@320 317 }
Chris@320 318 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
Chris@320 319
Chris@320 320 m_output = model;
Chris@320 321
Chris@441 322 } else {
Chris@320 323
Chris@441 324 // Anything that is not a 1D, note, or interval model and that
Chris@441 325 // has a fixed sample rate and more than one value per result
Chris@441 326 // must be a dense 3D model.
Chris@320 327
Chris@320 328 EditableDenseThreeDimensionalModel *model =
Chris@320 329 new EditableDenseThreeDimensionalModel
Chris@320 330 (modelRate, modelResolution, binCount, false);
Chris@320 331
Chris@320 332 if (!m_descriptor->binNames.empty()) {
Chris@320 333 std::vector<QString> names;
Chris@320 334 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
Chris@320 335 names.push_back(m_descriptor->binNames[i].c_str());
Chris@320 336 }
Chris@320 337 model->setBinNames(names);
Chris@320 338 }
Chris@320 339
Chris@320 340 m_output = model;
Chris@320 341 }
Chris@333 342
Chris@350 343 if (m_output) m_output->setSourceModel(input);
Chris@320 344 }
Chris@320 345
Chris@331 346 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 347 {
Chris@436 348 // std::cerr << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << std::endl;
Chris@320 349 delete m_plugin;
Chris@320 350 delete m_descriptor;
Chris@320 351 }
Chris@320 352
Chris@320 353 DenseTimeValueModel *
Chris@350 354 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 355 {
Chris@408 356 // std::cerr << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << std::endl;
Chris@408 357
Chris@320 358 DenseTimeValueModel *dtvm =
Chris@320 359 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 360 if (!dtvm) {
Chris@350 361 std::cerr << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
Chris@320 362 }
Chris@320 363 return dtvm;
Chris@320 364 }
Chris@320 365
Chris@320 366 void
Chris@331 367 FeatureExtractionModelTransformer::run()
Chris@320 368 {
Chris@350 369 DenseTimeValueModel *input = getConformingInput();
Chris@320 370 if (!input) return;
Chris@320 371
Chris@320 372 if (!m_output) return;
Chris@320 373
Chris@497 374 while (!input->isReady() && !m_abandoned) {
Chris@331 375 std::cerr << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << std::endl;
Chris@497 376 usleep(500000);
Chris@320 377 }
Chris@497 378 if (m_abandoned) return;
Chris@320 379
Chris@350 380 size_t sampleRate = input->getSampleRate();
Chris@320 381
Chris@320 382 size_t channelCount = input->getChannelCount();
Chris@320 383 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 384 channelCount = 1;
Chris@320 385 }
Chris@320 386
Chris@320 387 float **buffers = new float*[channelCount];
Chris@320 388 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 389 buffers[ch] = new float[m_transform.getBlockSize() + 2];
Chris@320 390 }
Chris@320 391
Chris@350 392 size_t stepSize = m_transform.getStepSize();
Chris@350 393 size_t blockSize = m_transform.getBlockSize();
Chris@350 394
Chris@320 395 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 396 Vamp::Plugin::FrequencyDomain);
Chris@320 397 std::vector<FFTModel *> fftModels;
Chris@320 398
Chris@320 399 if (frequencyDomain) {
Chris@320 400 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 401 FFTModel *model = new FFTModel
Chris@350 402 (getConformingInput(),
Chris@350 403 channelCount == 1 ? m_input.getChannel() : ch,
Chris@350 404 m_transform.getWindowType(),
Chris@350 405 blockSize,
Chris@350 406 stepSize,
Chris@350 407 blockSize,
Chris@334 408 false,
Chris@334 409 StorageAdviser::PrecisionCritical);
Chris@320 410 if (!model->isOK()) {
Chris@320 411 delete model;
Chris@320 412 setCompletion(100);
Chris@387 413 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@387 414 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
Chris@320 415 }
Chris@320 416 model->resume();
Chris@320 417 fftModels.push_back(model);
Chris@320 418 }
Chris@320 419 }
Chris@320 420
Chris@350 421 long startFrame = m_input.getModel()->getStartFrame();
Chris@350 422 long endFrame = m_input.getModel()->getEndFrame();
Chris@320 423
Chris@350 424 RealTime contextStartRT = m_transform.getStartTime();
Chris@350 425 RealTime contextDurationRT = m_transform.getDuration();
Chris@350 426
Chris@350 427 long contextStart =
Chris@350 428 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 429
Chris@350 430 long contextDuration =
Chris@350 431 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 432
Chris@320 433 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 434 contextStart = startFrame;
Chris@320 435 }
Chris@320 436
Chris@320 437 if (contextDuration == 0) {
Chris@320 438 contextDuration = endFrame - contextStart;
Chris@320 439 }
Chris@320 440 if (contextStart + contextDuration > endFrame) {
Chris@320 441 contextDuration = endFrame - contextStart;
Chris@320 442 }
Chris@320 443
Chris@320 444 long blockFrame = contextStart;
Chris@320 445
Chris@320 446 long prevCompletion = 0;
Chris@320 447
Chris@320 448 setCompletion(0);
Chris@320 449
Chris@320 450 while (!m_abandoned) {
Chris@320 451
Chris@320 452 if (frequencyDomain) {
Chris@350 453 if (blockFrame - int(blockSize)/2 >
Chris@320 454 contextStart + contextDuration) break;
Chris@320 455 } else {
Chris@320 456 if (blockFrame >=
Chris@320 457 contextStart + contextDuration) break;
Chris@320 458 }
Chris@320 459
Chris@331 460 // std::cerr << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@320 461 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@350 462 // << blockSize << std::endl;
Chris@320 463
Chris@320 464 long completion =
Chris@350 465 (((blockFrame - contextStart) / stepSize) * 99) /
Chris@350 466 (contextDuration / stepSize);
Chris@320 467
Chris@350 468 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 469
Chris@363 470 if (frequencyDomain) {
Chris@363 471 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 472 int column = (blockFrame - startFrame) / stepSize;
Chris@350 473 for (size_t i = 0; i <= blockSize/2; ++i) {
Chris@320 474 fftModels[ch]->getValuesAt
Chris@320 475 (column, i, buffers[ch][i*2], buffers[ch][i*2+1]);
Chris@320 476 }
Chris@363 477 }
Chris@363 478 } else {
Chris@363 479 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@320 480 }
Chris@320 481
Chris@497 482 if (m_abandoned) break;
Chris@497 483
Chris@320 484 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@320 485 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@320 486
Chris@497 487 if (m_abandoned) break;
Chris@497 488
Chris@320 489 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@320 490 Vamp::Plugin::Feature feature =
Chris@320 491 features[m_outputFeatureNo][fi];
Chris@320 492 addFeature(blockFrame, feature);
Chris@320 493 }
Chris@320 494
Chris@320 495 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@320 496 setCompletion(completion);
Chris@320 497 prevCompletion = completion;
Chris@320 498 }
Chris@320 499
Chris@350 500 blockFrame += stepSize;
Chris@320 501 }
Chris@320 502
Chris@497 503 if (!m_abandoned) {
Chris@497 504 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@320 505
Chris@497 506 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@497 507 Vamp::Plugin::Feature feature =
Chris@497 508 features[m_outputFeatureNo][fi];
Chris@497 509 addFeature(blockFrame, feature);
Chris@497 510 }
Chris@497 511 }
Chris@320 512
Chris@497 513 setCompletion(100);
Chris@320 514
Chris@320 515 if (frequencyDomain) {
Chris@320 516 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 517 delete fftModels[ch];
Chris@320 518 }
Chris@320 519 }
Chris@320 520 }
Chris@320 521
Chris@320 522 void
Chris@363 523 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@363 524 long startFrame, long size,
Chris@363 525 float **buffers)
Chris@320 526 {
Chris@320 527 long offset = 0;
Chris@320 528
Chris@320 529 if (startFrame < 0) {
Chris@363 530 for (int c = 0; c < channelCount; ++c) {
Chris@363 531 for (int i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 532 buffers[c][i] = 0.0f;
Chris@363 533 }
Chris@320 534 }
Chris@320 535 offset = -startFrame;
Chris@320 536 size -= offset;
Chris@320 537 if (size <= 0) return;
Chris@320 538 startFrame = 0;
Chris@320 539 }
Chris@320 540
Chris@350 541 DenseTimeValueModel *input = getConformingInput();
Chris@350 542 if (!input) return;
Chris@363 543
Chris@363 544 long got = 0;
Chris@350 545
Chris@363 546 if (channelCount == 1) {
Chris@363 547
Chris@363 548 got = input->getData(m_input.getChannel(), startFrame, size,
Chris@363 549 buffers[0] + offset);
Chris@363 550
Chris@363 551 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 552 // use mean instead of sum, as plugin input
Chris@363 553 float cc = float(input->getChannelCount());
Chris@363 554 for (long i = 0; i < size; ++i) {
Chris@363 555 buffers[0][i + offset] /= cc;
Chris@363 556 }
Chris@363 557 }
Chris@363 558
Chris@363 559 } else {
Chris@363 560
Chris@363 561 float **writebuf = buffers;
Chris@363 562 if (offset > 0) {
Chris@363 563 writebuf = new float *[channelCount];
Chris@363 564 for (int i = 0; i < channelCount; ++i) {
Chris@363 565 writebuf[i] = buffers[i] + offset;
Chris@363 566 }
Chris@363 567 }
Chris@363 568
Chris@363 569 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
Chris@363 570
Chris@363 571 if (writebuf != buffers) delete[] writebuf;
Chris@363 572 }
Chris@320 573
Chris@320 574 while (got < size) {
Chris@363 575 for (int c = 0; c < channelCount; ++c) {
Chris@363 576 buffers[c][got + offset] = 0.0;
Chris@363 577 }
Chris@320 578 ++got;
Chris@320 579 }
Chris@320 580 }
Chris@320 581
Chris@320 582 void
Chris@331 583 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
Chris@320 584 const Vamp::Plugin::Feature &feature)
Chris@320 585 {
Chris@350 586 size_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 587
Chris@331 588 // std::cerr << "FeatureExtractionModelTransformer::addFeature("
Chris@320 589 // << blockFrame << ")" << std::endl;
Chris@320 590
Chris@320 591 int binCount = 1;
Chris@320 592 if (m_descriptor->hasFixedBinCount) {
Chris@320 593 binCount = m_descriptor->binCount;
Chris@320 594 }
Chris@320 595
Chris@320 596 size_t frame = blockFrame;
Chris@320 597
Chris@320 598 if (m_descriptor->sampleType ==
Chris@320 599 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 600
Chris@320 601 if (!feature.hasTimestamp) {
Chris@320 602 std::cerr
Chris@331 603 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@320 604 << "Feature has variable sample rate but no timestamp!"
Chris@320 605 << std::endl;
Chris@320 606 return;
Chris@320 607 } else {
Chris@320 608 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@320 609 }
Chris@320 610
Chris@320 611 } else if (m_descriptor->sampleType ==
Chris@320 612 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 613
Chris@320 614 if (feature.hasTimestamp) {
Chris@320 615 //!!! warning: sampleRate may be non-integral
Chris@320 616 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
Chris@451 617 //!!! see comment above when setting up modelResolution and modelRate
Chris@451 618 // lrintf(m_descriptor->sampleRate));
Chris@451 619 inputRate);
Chris@320 620 } else {
Chris@320 621 frame = m_output->getEndFrame();
Chris@320 622 }
Chris@320 623 }
Chris@320 624
Chris@441 625 // Rather than repeat the complicated tests from the constructor
Chris@441 626 // to determine what sort of model we must be adding the features
Chris@441 627 // to, we instead test what sort of model the constructor decided
Chris@441 628 // to create.
Chris@320 629
Chris@441 630 if (isOutput<SparseOneDimensionalModel>()) {
Chris@441 631
Chris@441 632 SparseOneDimensionalModel *model =
Chris@350 633 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 634 if (!model) return;
Chris@350 635
Chris@441 636 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 637 (frame, feature.label.c_str()));
Chris@320 638
Chris@441 639 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 640
Chris@350 641 SparseTimeValueModel *model =
Chris@350 642 getConformingOutput<SparseTimeValueModel>();
Chris@320 643 if (!model) return;
Chris@350 644
Chris@454 645 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 646
Chris@454 647 float value = feature.values[i];
Chris@454 648
Chris@454 649 QString label = feature.label.c_str();
Chris@454 650 if (feature.values.size() > 1) {
Chris@454 651 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 652 }
Chris@454 653
Chris@454 654 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
Chris@454 655 }
Chris@320 656
Chris@441 657 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
Chris@320 658
Chris@441 659 int index = 0;
Chris@441 660
Chris@441 661 float value = 0.0;
Chris@441 662 if (feature.values.size() > index) {
Chris@441 663 value = feature.values[index++];
Chris@441 664 }
Chris@320 665
Chris@320 666 float duration = 1;
Chris@441 667 if (feature.hasDuration) {
Chris@441 668 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 669 } else {
Chris@441 670 if (feature.values.size() > index) {
Chris@441 671 duration = feature.values[index++];
Chris@441 672 }
Chris@441 673 }
Chris@320 674
Chris@441 675 if (isOutput<NoteModel>()) {
Chris@320 676
Chris@441 677 float velocity = 100;
Chris@441 678 if (feature.values.size() > index) {
Chris@441 679 velocity = feature.values[index++];
Chris@441 680 }
Chris@441 681 if (velocity < 0) velocity = 127;
Chris@441 682 if (velocity > 127) velocity = 127;
Chris@320 683
Chris@441 684 NoteModel *model = getConformingOutput<NoteModel>();
Chris@441 685 if (!model) return;
Chris@441 686 model->addPoint(NoteModel::Point(frame, value, // value is pitch
Chris@441 687 lrintf(duration),
Chris@441 688 velocity / 127.f,
Chris@441 689 feature.label.c_str()));
Chris@441 690 } else {
Chris@441 691 RegionModel *model = getConformingOutput<RegionModel>();
Chris@454 692 if (!model) return;
Chris@454 693
Chris@474 694 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 695
Chris@454 696 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 697
Chris@454 698 float value = feature.values[i];
Chris@454 699
Chris@454 700 QString label = feature.label.c_str();
Chris@454 701 if (feature.values.size() > 1) {
Chris@454 702 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 703 }
Chris@454 704
Chris@454 705 model->addPoint(RegionModel::Point(frame, value,
Chris@454 706 lrintf(duration),
Chris@454 707 label));
Chris@454 708 }
Chris@454 709 } else {
Chris@454 710
Chris@441 711 model->addPoint(RegionModel::Point(frame, value,
Chris@441 712 lrintf(duration),
Chris@441 713 feature.label.c_str()));
Chris@454 714 }
Chris@441 715 }
Chris@320 716
Chris@441 717 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 718
Chris@320 719 DenseThreeDimensionalModel::Column values = feature.values;
Chris@320 720
Chris@320 721 EditableDenseThreeDimensionalModel *model =
Chris@350 722 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 723 if (!model) return;
Chris@320 724
Chris@320 725 model->setColumn(frame / model->getResolution(), values);
Chris@441 726
Chris@441 727 } else {
Chris@441 728 std::cerr << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << std::endl;
Chris@320 729 }
Chris@320 730 }
Chris@320 731
Chris@320 732 void
Chris@331 733 FeatureExtractionModelTransformer::setCompletion(int completion)
Chris@320 734 {
Chris@320 735 int binCount = 1;
Chris@320 736 if (m_descriptor->hasFixedBinCount) {
Chris@320 737 binCount = m_descriptor->binCount;
Chris@320 738 }
Chris@320 739
Chris@331 740 // std::cerr << "FeatureExtractionModelTransformer::setCompletion("
Chris@320 741 // << completion << ")" << std::endl;
Chris@320 742
Chris@441 743 if (isOutput<SparseOneDimensionalModel>()) {
Chris@320 744
Chris@350 745 SparseOneDimensionalModel *model =
Chris@350 746 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 747 if (!model) return;
Chris@441 748 model->setCompletion(completion, true);
Chris@320 749
Chris@441 750 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 751
Chris@350 752 SparseTimeValueModel *model =
Chris@350 753 getConformingOutput<SparseTimeValueModel>();
Chris@320 754 if (!model) return;
Chris@441 755 model->setCompletion(completion, true);
Chris@320 756
Chris@441 757 } else if (isOutput<NoteModel>()) {
Chris@320 758
Chris@441 759 NoteModel *model = getConformingOutput<NoteModel>();
Chris@320 760 if (!model) return;
Chris@441 761 model->setCompletion(completion, true);
Chris@320 762
Chris@441 763 } else if (isOutput<RegionModel>()) {
Chris@441 764
Chris@441 765 RegionModel *model = getConformingOutput<RegionModel>();
Chris@441 766 if (!model) return;
Chris@441 767 model->setCompletion(completion, true);
Chris@441 768
Chris@441 769 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 770
Chris@320 771 EditableDenseThreeDimensionalModel *model =
Chris@350 772 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 773 if (!model) return;
Chris@350 774 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 775 }
Chris@320 776 }
Chris@320 777