annotate transform/FeatureExtractionPluginTransform.cpp @ 61:963e3187d920

* Fix slowness in serving FFT values to feature extraction plugin transform (failure to call resume() on FFT model) * Fix failure to update completion from time/value model
author Chris Cannam
date Tue, 17 Oct 2006 13:49:31 +0000
parents ca1e3f5657d5
children 51f34d378a3a
rev   line source
Chris@0 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@0 2
Chris@0 3 /*
Chris@0 4 Sonic Visualiser
Chris@0 5 An audio file viewer and annotation editor.
Chris@0 6 Centre for Digital Music, Queen Mary, University of London.
Chris@0 7 This file copyright 2006 Chris Cannam.
Chris@0 8
Chris@0 9 This program is free software; you can redistribute it and/or
Chris@0 10 modify it under the terms of the GNU General Public License as
Chris@0 11 published by the Free Software Foundation; either version 2 of the
Chris@0 12 License, or (at your option) any later version. See the file
Chris@0 13 COPYING included with this distribution for more information.
Chris@0 14 */
Chris@0 15
Chris@0 16 #include "FeatureExtractionPluginTransform.h"
Chris@0 17
Chris@0 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@0 19 #include "plugin/PluginXml.h"
Chris@0 20 #include "vamp-sdk/Plugin.h"
Chris@0 21
Chris@1 22 #include "data/model/Model.h"
Chris@0 23 #include "base/Window.h"
Chris@1 24 #include "data/model/SparseOneDimensionalModel.h"
Chris@1 25 #include "data/model/SparseTimeValueModel.h"
Chris@3 26 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@1 27 #include "data/model/DenseTimeValueModel.h"
Chris@1 28 #include "data/model/NoteModel.h"
Chris@3 29 #include "data/model/FFTModel.h"
Chris@55 30 #include "data/model/WaveFileModel.h"
Chris@0 31
Chris@0 32 #include <fftw3.h>
Chris@0 33
Chris@0 34 #include <iostream>
Chris@0 35
Chris@0 36 FeatureExtractionPluginTransform::FeatureExtractionPluginTransform(Model *inputModel,
Chris@0 37 QString pluginId,
Chris@27 38 const ExecutionContext &context,
Chris@0 39 QString configurationXml,
Chris@27 40 QString outputName) :
Chris@27 41 PluginTransform(inputModel, context),
Chris@0 42 m_plugin(0),
Chris@0 43 m_descriptor(0),
Chris@0 44 m_outputFeatureNo(0)
Chris@0 45 {
Chris@0 46 // std::cerr << "FeatureExtractionPluginTransform::FeatureExtractionPluginTransform: plugin " << pluginId.toStdString() << ", outputName " << outputName.toStdString() << std::endl;
Chris@0 47
Chris@0 48 FeatureExtractionPluginFactory *factory =
Chris@0 49 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@0 50
Chris@0 51 if (!factory) {
Chris@0 52 std::cerr << "FeatureExtractionPluginTransform: No factory available for plugin id \""
Chris@0 53 << pluginId.toStdString() << "\"" << std::endl;
Chris@0 54 return;
Chris@0 55 }
Chris@0 56
Chris@0 57 m_plugin = factory->instantiatePlugin(pluginId, m_input->getSampleRate());
Chris@0 58
Chris@0 59 if (!m_plugin) {
Chris@0 60 std::cerr << "FeatureExtractionPluginTransform: Failed to instantiate plugin \""
Chris@0 61 << pluginId.toStdString() << "\"" << std::endl;
Chris@0 62 return;
Chris@0 63 }
Chris@0 64
Chris@0 65 if (configurationXml != "") {
Chris@0 66 PluginXml(m_plugin).setParametersFromXml(configurationXml);
Chris@0 67 }
Chris@0 68
Chris@0 69 DenseTimeValueModel *input = getInput();
Chris@0 70 if (!input) return;
Chris@0 71
Chris@0 72 size_t channelCount = input->getChannelCount();
Chris@0 73 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@0 74 channelCount = 1;
Chris@0 75 }
Chris@0 76 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@0 77 std::cerr << "FeatureExtractionPluginTransform:: "
Chris@0 78 << "Can't provide enough channels to plugin (plugin min "
Chris@0 79 << m_plugin->getMinChannelCount() << ", max "
Chris@0 80 << m_plugin->getMaxChannelCount() << ", input model has "
Chris@0 81 << input->getChannelCount() << ")" << std::endl;
Chris@0 82 return;
Chris@0 83 }
Chris@0 84
Chris@27 85 std::cerr << "Initialising feature extraction plugin with channels = "
Chris@27 86 << channelCount << ", step = " << m_context.stepSize
Chris@27 87 << ", block = " << m_context.blockSize << std::endl;
Chris@27 88
Chris@27 89 if (!m_plugin->initialise(channelCount,
Chris@27 90 m_context.stepSize,
Chris@27 91 m_context.blockSize)) {
Chris@0 92 std::cerr << "FeatureExtractionPluginTransform: Plugin "
Chris@0 93 << m_plugin->getName() << " failed to initialise!" << std::endl;
Chris@0 94 return;
Chris@0 95 }
Chris@0 96
Chris@0 97 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@0 98
Chris@0 99 if (outputs.empty()) {
Chris@0 100 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
Chris@0 101 << pluginId.toStdString() << "\" has no outputs" << std::endl;
Chris@0 102 return;
Chris@0 103 }
Chris@0 104
Chris@0 105 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@0 106 if (outputName == "" || outputs[i].name == outputName.toStdString()) {
Chris@0 107 m_outputFeatureNo = i;
Chris@0 108 m_descriptor = new Vamp::Plugin::OutputDescriptor
Chris@0 109 (outputs[i]);
Chris@0 110 break;
Chris@0 111 }
Chris@0 112 }
Chris@0 113
Chris@0 114 if (!m_descriptor) {
Chris@0 115 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
Chris@0 116 << pluginId.toStdString() << "\" has no output named \""
Chris@0 117 << outputName.toStdString() << "\"" << std::endl;
Chris@0 118 return;
Chris@0 119 }
Chris@0 120
Chris@0 121 // std::cerr << "FeatureExtractionPluginTransform: output sample type "
Chris@0 122 // << m_descriptor->sampleType << std::endl;
Chris@0 123
Chris@0 124 int binCount = 1;
Chris@0 125 float minValue = 0.0, maxValue = 0.0;
Chris@0 126
Chris@0 127 if (m_descriptor->hasFixedBinCount) {
Chris@0 128 binCount = m_descriptor->binCount;
Chris@0 129 }
Chris@0 130
Chris@0 131 // std::cerr << "FeatureExtractionPluginTransform: output bin count "
Chris@0 132 // << binCount << std::endl;
Chris@0 133
Chris@0 134 if (binCount > 0 && m_descriptor->hasKnownExtents) {
Chris@0 135 minValue = m_descriptor->minValue;
Chris@0 136 maxValue = m_descriptor->maxValue;
Chris@0 137 }
Chris@0 138
Chris@0 139 size_t modelRate = m_input->getSampleRate();
Chris@0 140 size_t modelResolution = 1;
Chris@0 141
Chris@0 142 switch (m_descriptor->sampleType) {
Chris@0 143
Chris@0 144 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@0 145 if (m_descriptor->sampleRate != 0.0) {
Chris@0 146 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@0 147 }
Chris@0 148 break;
Chris@0 149
Chris@0 150 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@27 151 modelResolution = m_context.stepSize;
Chris@0 152 break;
Chris@0 153
Chris@0 154 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@0 155 modelRate = size_t(m_descriptor->sampleRate + 0.001);
Chris@0 156 break;
Chris@0 157 }
Chris@0 158
Chris@0 159 if (binCount == 0) {
Chris@0 160
Chris@0 161 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
Chris@0 162 false);
Chris@0 163
Chris@0 164 } else if (binCount == 1) {
Chris@0 165
Chris@0 166 SparseTimeValueModel *model = new SparseTimeValueModel
Chris@0 167 (modelRate, modelResolution, minValue, maxValue, false);
Chris@0 168 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
Chris@0 169
Chris@0 170 m_output = model;
Chris@0 171
Chris@0 172 } else if (m_descriptor->sampleType ==
Chris@0 173 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@0 174
Chris@0 175 // We don't have a sparse 3D model, so interpret this as a
Chris@0 176 // note model. There's nothing to define which values to use
Chris@0 177 // as which parameters of the note -- for the moment let's
Chris@0 178 // treat the first as pitch, second as duration in frames,
Chris@0 179 // third (if present) as velocity. (Our note model doesn't
Chris@0 180 // yet store velocity.)
Chris@0 181 //!!! todo: ask the user!
Chris@0 182
Chris@0 183 NoteModel *model = new NoteModel
Chris@0 184 (modelRate, modelResolution, minValue, maxValue, false);
Chris@0 185 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
Chris@0 186
Chris@0 187 m_output = model;
Chris@0 188
Chris@0 189 } else {
Chris@0 190
Chris@3 191 m_output = new EditableDenseThreeDimensionalModel
Chris@3 192 (modelRate, modelResolution, binCount, false);
Chris@0 193
Chris@0 194 if (!m_descriptor->binNames.empty()) {
Chris@0 195 std::vector<QString> names;
Chris@0 196 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
Chris@0 197 names.push_back(m_descriptor->binNames[i].c_str());
Chris@0 198 }
Chris@3 199 (dynamic_cast<EditableDenseThreeDimensionalModel *>(m_output))
Chris@0 200 ->setBinNames(names);
Chris@0 201 }
Chris@0 202 }
Chris@0 203 }
Chris@0 204
Chris@0 205 FeatureExtractionPluginTransform::~FeatureExtractionPluginTransform()
Chris@0 206 {
Chris@0 207 delete m_plugin;
Chris@0 208 delete m_descriptor;
Chris@0 209 }
Chris@0 210
Chris@0 211 DenseTimeValueModel *
Chris@0 212 FeatureExtractionPluginTransform::getInput()
Chris@0 213 {
Chris@0 214 DenseTimeValueModel *dtvm =
Chris@0 215 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@0 216 if (!dtvm) {
Chris@0 217 std::cerr << "FeatureExtractionPluginTransform::getInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
Chris@0 218 }
Chris@0 219 return dtvm;
Chris@0 220 }
Chris@0 221
Chris@0 222 void
Chris@0 223 FeatureExtractionPluginTransform::run()
Chris@0 224 {
Chris@0 225 DenseTimeValueModel *input = getInput();
Chris@0 226 if (!input) return;
Chris@0 227
Chris@55 228 while (!input->isReady()) {
Chris@55 229 if (dynamic_cast<WaveFileModel *>(input)) break; // no need to wait
Chris@55 230 std::cerr << "FeatureExtractionPluginTransform::run: Waiting for input model to be ready..." << std::endl;
Chris@55 231 sleep(1);
Chris@55 232 }
Chris@55 233
Chris@0 234 if (!m_output) return;
Chris@0 235
Chris@0 236 size_t sampleRate = m_input->getSampleRate();
Chris@0 237
Chris@0 238 size_t channelCount = input->getChannelCount();
Chris@0 239 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@0 240 channelCount = 1;
Chris@0 241 }
Chris@0 242
Chris@0 243 float **buffers = new float*[channelCount];
Chris@0 244 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@27 245 buffers[ch] = new float[m_context.blockSize];
Chris@0 246 }
Chris@0 247
Chris@0 248 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@0 249 Vamp::Plugin::FrequencyDomain);
Chris@3 250 std::vector<FFTModel *> fftModels;
Chris@0 251
Chris@0 252 if (frequencyDomain) {
Chris@0 253 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@61 254 FFTModel *model = new FFTModel
Chris@0 255 (getInput(),
Chris@27 256 channelCount == 1 ? m_context.channel : ch,
Chris@27 257 m_context.windowType,
Chris@27 258 m_context.blockSize,
Chris@27 259 m_context.stepSize,
Chris@27 260 m_context.blockSize,
Chris@61 261 false);
Chris@61 262 model->resume();
Chris@61 263 fftModels.push_back(model);
Chris@0 264 }
Chris@0 265 }
Chris@0 266
Chris@0 267 long startFrame = m_input->getStartFrame();
Chris@0 268 long endFrame = m_input->getEndFrame();
Chris@0 269 long blockFrame = startFrame;
Chris@0 270
Chris@0 271 long prevCompletion = 0;
Chris@0 272
Chris@0 273 while (1) {
Chris@0 274
Chris@0 275 if (frequencyDomain) {
Chris@27 276 if (blockFrame - int(m_context.blockSize)/2 > endFrame) break;
Chris@0 277 } else {
Chris@0 278 if (blockFrame >= endFrame) break;
Chris@0 279 }
Chris@0 280
Chris@33 281 // std::cerr << "FeatureExtractionPluginTransform::run: blockFrame "
Chris@33 282 // << blockFrame << std::endl;
Chris@0 283
Chris@0 284 long completion =
Chris@27 285 (((blockFrame - startFrame) / m_context.stepSize) * 99) /
Chris@27 286 ( (endFrame - startFrame) / m_context.stepSize);
Chris@0 287
Chris@0 288 // channelCount is either m_input->channelCount or 1
Chris@0 289
Chris@0 290 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@0 291 if (frequencyDomain) {
Chris@27 292 int column = (blockFrame - startFrame) / m_context.stepSize;
Chris@27 293 for (size_t i = 0; i < m_context.blockSize/2; ++i) {
Chris@3 294 fftModels[ch]->getValuesAt
Chris@0 295 (column, i, buffers[ch][i*2], buffers[ch][i*2+1]);
Chris@0 296 }
Chris@0 297 } else {
Chris@0 298 getFrames(ch, channelCount,
Chris@27 299 blockFrame, m_context.blockSize, buffers[ch]);
Chris@0 300 }
Chris@0 301 }
Chris@0 302
Chris@0 303 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@0 304 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@0 305
Chris@0 306 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@0 307 Vamp::Plugin::Feature feature =
Chris@0 308 features[m_outputFeatureNo][fi];
Chris@0 309 addFeature(blockFrame, feature);
Chris@0 310 }
Chris@0 311
Chris@0 312 if (blockFrame == startFrame || completion > prevCompletion) {
Chris@0 313 setCompletion(completion);
Chris@0 314 prevCompletion = completion;
Chris@0 315 }
Chris@0 316
Chris@27 317 blockFrame += m_context.stepSize;
Chris@0 318 }
Chris@0 319
Chris@0 320 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@0 321
Chris@0 322 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@0 323 Vamp::Plugin::Feature feature =
Chris@0 324 features[m_outputFeatureNo][fi];
Chris@0 325 addFeature(blockFrame, feature);
Chris@0 326 }
Chris@0 327
Chris@0 328 if (frequencyDomain) {
Chris@0 329 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@3 330 delete fftModels[ch];
Chris@0 331 }
Chris@0 332 }
Chris@0 333
Chris@0 334 setCompletion(100);
Chris@0 335 }
Chris@0 336
Chris@0 337 void
Chris@0 338 FeatureExtractionPluginTransform::getFrames(int channel, int channelCount,
Chris@0 339 long startFrame, long size,
Chris@0 340 float *buffer)
Chris@0 341 {
Chris@0 342 long offset = 0;
Chris@0 343
Chris@0 344 if (startFrame < 0) {
Chris@0 345 for (int i = 0; i < size && startFrame + i < 0; ++i) {
Chris@0 346 buffer[i] = 0.0f;
Chris@0 347 }
Chris@0 348 offset = -startFrame;
Chris@0 349 size -= offset;
Chris@0 350 if (size <= 0) return;
Chris@0 351 startFrame = 0;
Chris@0 352 }
Chris@0 353
Chris@0 354 long got = getInput()->getValues
Chris@27 355 ((channelCount == 1 ? m_context.channel : channel),
Chris@0 356 startFrame, startFrame + size, buffer + offset);
Chris@0 357
Chris@0 358 while (got < size) {
Chris@0 359 buffer[offset + got] = 0.0;
Chris@0 360 ++got;
Chris@0 361 }
Chris@0 362
Chris@27 363 if (m_context.channel == -1 && channelCount == 1 &&
Chris@0 364 getInput()->getChannelCount() > 1) {
Chris@0 365 // use mean instead of sum, as plugin input
Chris@0 366 int cc = getInput()->getChannelCount();
Chris@0 367 for (long i = 0; i < size; ++i) {
Chris@0 368 buffer[i] /= cc;
Chris@0 369 }
Chris@0 370 }
Chris@0 371 }
Chris@0 372
Chris@0 373 void
Chris@0 374 FeatureExtractionPluginTransform::addFeature(size_t blockFrame,
Chris@0 375 const Vamp::Plugin::Feature &feature)
Chris@0 376 {
Chris@0 377 size_t inputRate = m_input->getSampleRate();
Chris@0 378
Chris@0 379 // std::cerr << "FeatureExtractionPluginTransform::addFeature("
Chris@0 380 // << blockFrame << ")" << std::endl;
Chris@0 381
Chris@0 382 int binCount = 1;
Chris@0 383 if (m_descriptor->hasFixedBinCount) {
Chris@0 384 binCount = m_descriptor->binCount;
Chris@0 385 }
Chris@0 386
Chris@0 387 size_t frame = blockFrame;
Chris@0 388
Chris@0 389 if (m_descriptor->sampleType ==
Chris@0 390 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@0 391
Chris@0 392 if (!feature.hasTimestamp) {
Chris@0 393 std::cerr
Chris@0 394 << "WARNING: FeatureExtractionPluginTransform::addFeature: "
Chris@0 395 << "Feature has variable sample rate but no timestamp!"
Chris@0 396 << std::endl;
Chris@0 397 return;
Chris@0 398 } else {
Chris@0 399 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@0 400 }
Chris@0 401
Chris@0 402 } else if (m_descriptor->sampleType ==
Chris@0 403 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@0 404
Chris@0 405 if (feature.hasTimestamp) {
Chris@0 406 //!!! warning: sampleRate may be non-integral
Chris@0 407 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
Chris@0 408 m_descriptor->sampleRate);
Chris@0 409 } else {
Chris@50 410 frame = m_output->getEndFrame();
Chris@0 411 }
Chris@0 412 }
Chris@0 413
Chris@0 414 if (binCount == 0) {
Chris@0 415
Chris@0 416 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
Chris@0 417 if (!model) return;
Chris@0 418 model->addPoint(SparseOneDimensionalModel::Point(frame, feature.label.c_str()));
Chris@0 419
Chris@0 420 } else if (binCount == 1) {
Chris@0 421
Chris@0 422 float value = 0.0;
Chris@0 423 if (feature.values.size() > 0) value = feature.values[0];
Chris@0 424
Chris@0 425 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
Chris@0 426 if (!model) return;
Chris@0 427 model->addPoint(SparseTimeValueModel::Point(frame, value, feature.label.c_str()));
Chris@0 428
Chris@0 429 } else if (m_descriptor->sampleType ==
Chris@0 430 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@0 431
Chris@0 432 float pitch = 0.0;
Chris@0 433 if (feature.values.size() > 0) pitch = feature.values[0];
Chris@0 434
Chris@0 435 float duration = 1;
Chris@0 436 if (feature.values.size() > 1) duration = feature.values[1];
Chris@0 437
Chris@0 438 float velocity = 100;
Chris@0 439 if (feature.values.size() > 2) velocity = feature.values[2];
Chris@0 440
Chris@0 441 NoteModel *model = getOutput<NoteModel>();
Chris@0 442 if (!model) return;
Chris@0 443
Chris@0 444 model->addPoint(NoteModel::Point(frame, pitch, duration, feature.label.c_str()));
Chris@0 445
Chris@0 446 } else {
Chris@0 447
Chris@51 448 DenseThreeDimensionalModel::Column values = feature.values;
Chris@0 449
Chris@3 450 EditableDenseThreeDimensionalModel *model =
Chris@3 451 getOutput<EditableDenseThreeDimensionalModel>();
Chris@0 452 if (!model) return;
Chris@0 453
Chris@51 454 model->setColumn(frame / model->getResolution(), values);
Chris@0 455 }
Chris@0 456 }
Chris@0 457
Chris@0 458 void
Chris@0 459 FeatureExtractionPluginTransform::setCompletion(int completion)
Chris@0 460 {
Chris@0 461 int binCount = 1;
Chris@0 462 if (m_descriptor->hasFixedBinCount) {
Chris@0 463 binCount = m_descriptor->binCount;
Chris@0 464 }
Chris@0 465
Chris@33 466 std::cerr << "FeatureExtractionPluginTransform::setCompletion("
Chris@33 467 << completion << ")" << std::endl;
Chris@33 468
Chris@0 469 if (binCount == 0) {
Chris@0 470
Chris@0 471 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
Chris@0 472 if (!model) return;
Chris@0 473 model->setCompletion(completion);
Chris@0 474
Chris@0 475 } else if (binCount == 1) {
Chris@0 476
Chris@0 477 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
Chris@0 478 if (!model) return;
Chris@0 479 model->setCompletion(completion);
Chris@0 480
Chris@0 481 } else if (m_descriptor->sampleType ==
Chris@0 482 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@0 483
Chris@0 484 NoteModel *model = getOutput<NoteModel>();
Chris@0 485 if (!model) return;
Chris@0 486 model->setCompletion(completion);
Chris@0 487
Chris@0 488 } else {
Chris@0 489
Chris@3 490 EditableDenseThreeDimensionalModel *model =
Chris@3 491 getOutput<EditableDenseThreeDimensionalModel>();
Chris@0 492 if (!model) return;
Chris@0 493 model->setCompletion(completion);
Chris@0 494 }
Chris@0 495 }
Chris@0 496