annotate transform/FeatureExtractionModelTransformer.cpp @ 537:3cc4b7cd2aa5

* Merge from one-fftdataserver-per-fftmodel branch. This bit of reworking (which is not described very accurately by the title of the branch) turns the MatrixFile object into something that either reads or writes, but not both, and separates the FFT file cache reader and writer implementations separately. This allows the FFT data server to have a single thread owning writers and one reader per "customer" thread, and for all locking to be vastly simplified and concentrated in the data server alone (because none of the classes it makes use of is used in more than one thread at a time). The result is faster and more trustworthy code.
author Chris Cannam
date Tue, 27 Jan 2009 13:25:10 +0000
parents 3ccf48fb81d6
children 53e5dc8439e7
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@320 19 #include "plugin/PluginXml.h"
Chris@475 20 #include <vamp-hostsdk/Plugin.h>
Chris@320 21
Chris@320 22 #include "data/model/Model.h"
Chris@320 23 #include "base/Window.h"
Chris@387 24 #include "base/Exceptions.h"
Chris@320 25 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 26 #include "data/model/SparseTimeValueModel.h"
Chris@320 27 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 28 #include "data/model/DenseTimeValueModel.h"
Chris@320 29 #include "data/model/NoteModel.h"
Chris@441 30 #include "data/model/RegionModel.h"
Chris@320 31 #include "data/model/FFTModel.h"
Chris@320 32 #include "data/model/WaveFileModel.h"
Chris@320 33
Chris@350 34 #include "TransformFactory.h"
Chris@350 35
Chris@320 36 #include <iostream>
Chris@320 37
Chris@350 38 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@350 39 const Transform &transform) :
Chris@350 40 ModelTransformer(in, transform),
Chris@320 41 m_plugin(0),
Chris@320 42 m_descriptor(0),
Chris@320 43 m_outputFeatureNo(0)
Chris@320 44 {
Chris@350 45 // std::cerr << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId.toStdString() << ", outputName " << m_transform.getOutput().toStdString() << std::endl;
Chris@350 46
Chris@350 47 QString pluginId = transform.getPluginIdentifier();
Chris@320 48
Chris@320 49 FeatureExtractionPluginFactory *factory =
Chris@320 50 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@320 51
Chris@320 52 if (!factory) {
Chris@361 53 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@320 54 return;
Chris@320 55 }
Chris@320 56
Chris@350 57 DenseTimeValueModel *input = getConformingInput();
Chris@350 58 if (!input) {
Chris@361 59 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@350 60 return;
Chris@350 61 }
Chris@320 62
Chris@350 63 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 64 if (!m_plugin) {
Chris@361 65 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@320 66 return;
Chris@320 67 }
Chris@320 68
Chris@350 69 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@350 70 (m_transform, m_plugin);
Chris@343 71
Chris@350 72 TransformFactory::getInstance()->setPluginParameters
Chris@350 73 (m_transform, m_plugin);
Chris@320 74
Chris@320 75 size_t channelCount = input->getChannelCount();
Chris@320 76 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 77 channelCount = 1;
Chris@320 78 }
Chris@320 79 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@361 80 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 81 .arg(pluginId)
Chris@361 82 .arg(m_plugin->getMinChannelCount())
Chris@361 83 .arg(m_plugin->getMaxChannelCount())
Chris@361 84 .arg(input->getChannelCount());
Chris@320 85 return;
Chris@320 86 }
Chris@320 87
Chris@320 88 std::cerr << "Initialising feature extraction plugin with channels = "
Chris@350 89 << channelCount << ", step = " << m_transform.getStepSize()
Chris@350 90 << ", block = " << m_transform.getBlockSize() << std::endl;
Chris@320 91
Chris@320 92 if (!m_plugin->initialise(channelCount,
Chris@350 93 m_transform.getStepSize(),
Chris@350 94 m_transform.getBlockSize())) {
Chris@361 95
Chris@361 96 size_t pstep = m_transform.getStepSize();
Chris@361 97 size_t pblock = m_transform.getBlockSize();
Chris@361 98
Chris@361 99 m_transform.setStepSize(0);
Chris@361 100 m_transform.setBlockSize(0);
Chris@361 101 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@361 102 (m_transform, m_plugin);
Chris@361 103
Chris@361 104 if (m_transform.getStepSize() != pstep ||
Chris@361 105 m_transform.getBlockSize() != pblock) {
Chris@361 106
Chris@361 107 if (!m_plugin->initialise(channelCount,
Chris@361 108 m_transform.getStepSize(),
Chris@361 109 m_transform.getBlockSize())) {
Chris@361 110
Chris@361 111 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 112 return;
Chris@361 113
Chris@361 114 } else {
Chris@361 115
Chris@361 116 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 117 .arg(pluginId)
Chris@361 118 .arg(pstep)
Chris@361 119 .arg(pblock)
Chris@361 120 .arg(m_transform.getStepSize())
Chris@361 121 .arg(m_transform.getBlockSize());
Chris@361 122 }
Chris@361 123
Chris@361 124 } else {
Chris@361 125
Chris@361 126 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 127 return;
Chris@361 128 }
Chris@320 129 }
Chris@320 130
Chris@366 131 if (m_transform.getPluginVersion() != "") {
Chris@366 132 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@366 133 if (pv != m_transform.getPluginVersion()) {
Chris@366 134 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@366 135 .arg(m_transform.getPluginVersion())
Chris@366 136 .arg(pluginId)
Chris@366 137 .arg(pv);
Chris@366 138 if (m_message != "") {
Chris@366 139 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 140 } else {
Chris@366 141 m_message = vm;
Chris@366 142 }
Chris@366 143 }
Chris@366 144 }
Chris@366 145
Chris@320 146 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 147
Chris@320 148 if (outputs.empty()) {
Chris@361 149 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@320 150 return;
Chris@320 151 }
Chris@320 152
Chris@320 153 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@429 154 // std::cerr << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput().toStdString() << "\"" << std::endl;
Chris@350 155 if (m_transform.getOutput() == "" ||
Chris@350 156 outputs[i].identifier == m_transform.getOutput().toStdString()) {
Chris@320 157 m_outputFeatureNo = i;
Chris@441 158 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
Chris@320 159 break;
Chris@320 160 }
Chris@320 161 }
Chris@320 162
Chris@320 163 if (!m_descriptor) {
Chris@361 164 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@361 165 .arg(pluginId)
Chris@361 166 .arg(m_transform.getOutput());
Chris@320 167 return;
Chris@320 168 }
Chris@320 169
Chris@331 170 // std::cerr << "FeatureExtractionModelTransformer: output sample type "
Chris@320 171 // << m_descriptor->sampleType << std::endl;
Chris@320 172
Chris@320 173 int binCount = 1;
Chris@320 174 float minValue = 0.0, maxValue = 0.0;
Chris@320 175 bool haveExtents = false;
Chris@320 176
Chris@320 177 if (m_descriptor->hasFixedBinCount) {
Chris@320 178 binCount = m_descriptor->binCount;
Chris@320 179 }
Chris@320 180
Chris@331 181 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@320 182 // << binCount << std::endl;
Chris@320 183
Chris@320 184 if (binCount > 0 && m_descriptor->hasKnownExtents) {
Chris@320 185 minValue = m_descriptor->minValue;
Chris@320 186 maxValue = m_descriptor->maxValue;
Chris@320 187 haveExtents = true;
Chris@320 188 }
Chris@320 189
Chris@350 190 size_t modelRate = input->getSampleRate();
Chris@320 191 size_t modelResolution = 1;
Chris@320 192
Chris@320 193 switch (m_descriptor->sampleType) {
Chris@320 194
Chris@320 195 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@320 196 if (m_descriptor->sampleRate != 0.0) {
Chris@320 197 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@320 198 }
Chris@320 199 break;
Chris@320 200
Chris@320 201 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@350 202 modelResolution = m_transform.getStepSize();
Chris@320 203 break;
Chris@320 204
Chris@320 205 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@451 206 //!!! SV doesn't actually support display of models that have
Chris@451 207 //!!! different underlying rates together -- so we always set
Chris@451 208 //!!! the model rate to be the input model's rate, and adjust
Chris@451 209 //!!! the resolution appropriately. We can't properly display
Chris@451 210 //!!! data with a higher resolution than the base model at all
Chris@451 211 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
Chris@451 212 if (m_descriptor->sampleRate > input->getSampleRate()) {
Chris@451 213 modelResolution = 1;
Chris@451 214 } else {
Chris@451 215 modelResolution = size_t(input->getSampleRate() /
Chris@451 216 m_descriptor->sampleRate);
Chris@451 217 }
Chris@320 218 break;
Chris@320 219 }
Chris@320 220
Chris@441 221 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 222
Chris@441 223 if (binCount == 0 &&
Chris@441 224 (preDurationPlugin || !m_descriptor->hasDuration)) {
Chris@320 225
Chris@445 226 // Anything with no value and no duration is an instant
Chris@445 227
Chris@320 228 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
Chris@320 229 false);
Chris@320 230
Chris@441 231 } else if ((preDurationPlugin && binCount > 1 &&
Chris@441 232 (m_descriptor->sampleType ==
Chris@441 233 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@441 234 (!preDurationPlugin && m_descriptor->hasDuration)) {
Chris@441 235
Chris@441 236 // For plugins using the old v1 API without explicit duration,
Chris@441 237 // we treat anything that has multiple bins (i.e. that has the
Chris@441 238 // potential to have value and duration) and a variable sample
Chris@441 239 // rate as a note model, taking its values as pitch, duration
Chris@441 240 // and velocity (if present) respectively. This is the same
Chris@441 241 // behaviour as always applied by SV to these plugins in the
Chris@441 242 // past.
Chris@441 243
Chris@441 244 // For plugins with the newer API, we treat anything with
Chris@441 245 // duration as either a note model with pitch and velocity, or
Chris@441 246 // a region model.
Chris@441 247
Chris@441 248 // How do we know whether it's an interval or note model?
Chris@441 249 // What's the essential difference? Is a note model any
Chris@441 250 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 251 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 252 // note model always have velocity? This is a good question
Chris@441 253 // to be addressed by accompanying RDF, but for the moment we
Chris@441 254 // will do the following...
Chris@441 255
Chris@441 256 bool isNoteModel = false;
Chris@441 257
Chris@441 258 // Regions have only value (and duration -- we can't extract a
Chris@441 259 // region model from an old-style plugin that doesn't support
Chris@441 260 // duration)
Chris@441 261 if (binCount > 1) isNoteModel = true;
Chris@441 262
Chris@441 263 // Regions do not have units of Hz (a sweeping assumption!)
Chris@441 264 if (m_descriptor->unit == "Hz") isNoteModel = true;
Chris@441 265
Chris@441 266 // If we had a "sparse 3D model", we would have the additional
Chris@441 267 // problem of determining whether to use that here (if bin
Chris@441 268 // count > 1). But we don't.
Chris@441 269
Chris@441 270 if (isNoteModel) {
Chris@441 271
Chris@441 272 NoteModel *model;
Chris@441 273 if (haveExtents) {
Chris@441 274 model = new NoteModel
Chris@441 275 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 276 } else {
Chris@441 277 model = new NoteModel
Chris@441 278 (modelRate, modelResolution, false);
Chris@441 279 }
Chris@441 280 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 281 m_output = model;
Chris@441 282
Chris@441 283 } else {
Chris@441 284
Chris@441 285 RegionModel *model;
Chris@441 286 if (haveExtents) {
Chris@441 287 model = new RegionModel
Chris@441 288 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 289 } else {
Chris@441 290 model = new RegionModel
Chris@441 291 (modelRate, modelResolution, false);
Chris@441 292 }
Chris@441 293 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 294 m_output = model;
Chris@441 295 }
Chris@441 296
Chris@441 297 } else if (binCount == 1 ||
Chris@441 298 (m_descriptor->sampleType ==
Chris@441 299 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 300
Chris@441 301 // Anything that is not a 1D, note, or interval model and that
Chris@441 302 // has only one value per result must be a sparse time value
Chris@441 303 // model.
Chris@441 304
Chris@441 305 // Anything that is not a 1D, note, or interval model and that
Chris@441 306 // has a variable sample rate is also treated as a sparse time
Chris@441 307 // value model regardless of its bin count, because we lack a
Chris@441 308 // sparse 3D model.
Chris@320 309
Chris@320 310 SparseTimeValueModel *model;
Chris@320 311 if (haveExtents) {
Chris@320 312 model = new SparseTimeValueModel
Chris@320 313 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 314 } else {
Chris@320 315 model = new SparseTimeValueModel
Chris@320 316 (modelRate, modelResolution, false);
Chris@320 317 }
Chris@320 318 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
Chris@320 319
Chris@320 320 m_output = model;
Chris@320 321
Chris@441 322 } else {
Chris@320 323
Chris@441 324 // Anything that is not a 1D, note, or interval model and that
Chris@441 325 // has a fixed sample rate and more than one value per result
Chris@441 326 // must be a dense 3D model.
Chris@320 327
Chris@320 328 EditableDenseThreeDimensionalModel *model =
Chris@320 329 new EditableDenseThreeDimensionalModel
Chris@535 330 (modelRate, modelResolution, binCount,
Chris@535 331 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
Chris@535 332 false);
Chris@320 333
Chris@320 334 if (!m_descriptor->binNames.empty()) {
Chris@320 335 std::vector<QString> names;
Chris@320 336 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
Chris@320 337 names.push_back(m_descriptor->binNames[i].c_str());
Chris@320 338 }
Chris@320 339 model->setBinNames(names);
Chris@320 340 }
Chris@320 341
Chris@320 342 m_output = model;
Chris@320 343 }
Chris@333 344
Chris@350 345 if (m_output) m_output->setSourceModel(input);
Chris@320 346 }
Chris@320 347
Chris@331 348 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 349 {
Chris@436 350 // std::cerr << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << std::endl;
Chris@320 351 delete m_plugin;
Chris@320 352 delete m_descriptor;
Chris@320 353 }
Chris@320 354
Chris@320 355 DenseTimeValueModel *
Chris@350 356 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 357 {
Chris@408 358 // std::cerr << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << std::endl;
Chris@408 359
Chris@320 360 DenseTimeValueModel *dtvm =
Chris@320 361 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 362 if (!dtvm) {
Chris@350 363 std::cerr << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
Chris@320 364 }
Chris@320 365 return dtvm;
Chris@320 366 }
Chris@320 367
Chris@320 368 void
Chris@331 369 FeatureExtractionModelTransformer::run()
Chris@320 370 {
Chris@350 371 DenseTimeValueModel *input = getConformingInput();
Chris@320 372 if (!input) return;
Chris@320 373
Chris@320 374 if (!m_output) return;
Chris@320 375
Chris@497 376 while (!input->isReady() && !m_abandoned) {
Chris@331 377 std::cerr << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << std::endl;
Chris@497 378 usleep(500000);
Chris@320 379 }
Chris@497 380 if (m_abandoned) return;
Chris@320 381
Chris@350 382 size_t sampleRate = input->getSampleRate();
Chris@320 383
Chris@320 384 size_t channelCount = input->getChannelCount();
Chris@320 385 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 386 channelCount = 1;
Chris@320 387 }
Chris@320 388
Chris@320 389 float **buffers = new float*[channelCount];
Chris@320 390 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 391 buffers[ch] = new float[m_transform.getBlockSize() + 2];
Chris@320 392 }
Chris@320 393
Chris@350 394 size_t stepSize = m_transform.getStepSize();
Chris@350 395 size_t blockSize = m_transform.getBlockSize();
Chris@350 396
Chris@320 397 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 398 Vamp::Plugin::FrequencyDomain);
Chris@320 399 std::vector<FFTModel *> fftModels;
Chris@320 400
Chris@320 401 if (frequencyDomain) {
Chris@320 402 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 403 FFTModel *model = new FFTModel
Chris@350 404 (getConformingInput(),
Chris@350 405 channelCount == 1 ? m_input.getChannel() : ch,
Chris@350 406 m_transform.getWindowType(),
Chris@350 407 blockSize,
Chris@350 408 stepSize,
Chris@350 409 blockSize,
Chris@334 410 false,
Chris@334 411 StorageAdviser::PrecisionCritical);
Chris@320 412 if (!model->isOK()) {
Chris@320 413 delete model;
Chris@320 414 setCompletion(100);
Chris@387 415 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@387 416 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
Chris@320 417 }
Chris@320 418 model->resume();
Chris@320 419 fftModels.push_back(model);
Chris@320 420 }
Chris@320 421 }
Chris@320 422
Chris@350 423 long startFrame = m_input.getModel()->getStartFrame();
Chris@350 424 long endFrame = m_input.getModel()->getEndFrame();
Chris@320 425
Chris@350 426 RealTime contextStartRT = m_transform.getStartTime();
Chris@350 427 RealTime contextDurationRT = m_transform.getDuration();
Chris@350 428
Chris@350 429 long contextStart =
Chris@350 430 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 431
Chris@350 432 long contextDuration =
Chris@350 433 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 434
Chris@320 435 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 436 contextStart = startFrame;
Chris@320 437 }
Chris@320 438
Chris@320 439 if (contextDuration == 0) {
Chris@320 440 contextDuration = endFrame - contextStart;
Chris@320 441 }
Chris@320 442 if (contextStart + contextDuration > endFrame) {
Chris@320 443 contextDuration = endFrame - contextStart;
Chris@320 444 }
Chris@320 445
Chris@320 446 long blockFrame = contextStart;
Chris@320 447
Chris@320 448 long prevCompletion = 0;
Chris@320 449
Chris@320 450 setCompletion(0);
Chris@320 451
Chris@320 452 while (!m_abandoned) {
Chris@320 453
Chris@320 454 if (frequencyDomain) {
Chris@350 455 if (blockFrame - int(blockSize)/2 >
Chris@320 456 contextStart + contextDuration) break;
Chris@320 457 } else {
Chris@320 458 if (blockFrame >=
Chris@320 459 contextStart + contextDuration) break;
Chris@320 460 }
Chris@320 461
Chris@331 462 // std::cerr << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@320 463 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@350 464 // << blockSize << std::endl;
Chris@320 465
Chris@320 466 long completion =
Chris@350 467 (((blockFrame - contextStart) / stepSize) * 99) /
Chris@350 468 (contextDuration / stepSize);
Chris@320 469
Chris@350 470 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 471
Chris@363 472 if (frequencyDomain) {
Chris@363 473 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 474 int column = (blockFrame - startFrame) / stepSize;
Chris@350 475 for (size_t i = 0; i <= blockSize/2; ++i) {
Chris@320 476 fftModels[ch]->getValuesAt
Chris@320 477 (column, i, buffers[ch][i*2], buffers[ch][i*2+1]);
Chris@320 478 }
Chris@363 479 }
Chris@363 480 } else {
Chris@363 481 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@320 482 }
Chris@320 483
Chris@497 484 if (m_abandoned) break;
Chris@497 485
Chris@320 486 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@320 487 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@320 488
Chris@497 489 if (m_abandoned) break;
Chris@497 490
Chris@320 491 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@320 492 Vamp::Plugin::Feature feature =
Chris@320 493 features[m_outputFeatureNo][fi];
Chris@320 494 addFeature(blockFrame, feature);
Chris@320 495 }
Chris@320 496
Chris@320 497 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@320 498 setCompletion(completion);
Chris@320 499 prevCompletion = completion;
Chris@320 500 }
Chris@320 501
Chris@350 502 blockFrame += stepSize;
Chris@320 503 }
Chris@320 504
Chris@497 505 if (!m_abandoned) {
Chris@497 506 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@320 507
Chris@497 508 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@497 509 Vamp::Plugin::Feature feature =
Chris@497 510 features[m_outputFeatureNo][fi];
Chris@497 511 addFeature(blockFrame, feature);
Chris@497 512 }
Chris@497 513 }
Chris@320 514
Chris@497 515 setCompletion(100);
Chris@320 516
Chris@320 517 if (frequencyDomain) {
Chris@320 518 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 519 delete fftModels[ch];
Chris@320 520 }
Chris@320 521 }
Chris@320 522 }
Chris@320 523
Chris@320 524 void
Chris@363 525 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@363 526 long startFrame, long size,
Chris@363 527 float **buffers)
Chris@320 528 {
Chris@320 529 long offset = 0;
Chris@320 530
Chris@320 531 if (startFrame < 0) {
Chris@363 532 for (int c = 0; c < channelCount; ++c) {
Chris@363 533 for (int i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 534 buffers[c][i] = 0.0f;
Chris@363 535 }
Chris@320 536 }
Chris@320 537 offset = -startFrame;
Chris@320 538 size -= offset;
Chris@320 539 if (size <= 0) return;
Chris@320 540 startFrame = 0;
Chris@320 541 }
Chris@320 542
Chris@350 543 DenseTimeValueModel *input = getConformingInput();
Chris@350 544 if (!input) return;
Chris@363 545
Chris@363 546 long got = 0;
Chris@350 547
Chris@363 548 if (channelCount == 1) {
Chris@363 549
Chris@363 550 got = input->getData(m_input.getChannel(), startFrame, size,
Chris@363 551 buffers[0] + offset);
Chris@363 552
Chris@363 553 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 554 // use mean instead of sum, as plugin input
Chris@363 555 float cc = float(input->getChannelCount());
Chris@363 556 for (long i = 0; i < size; ++i) {
Chris@363 557 buffers[0][i + offset] /= cc;
Chris@363 558 }
Chris@363 559 }
Chris@363 560
Chris@363 561 } else {
Chris@363 562
Chris@363 563 float **writebuf = buffers;
Chris@363 564 if (offset > 0) {
Chris@363 565 writebuf = new float *[channelCount];
Chris@363 566 for (int i = 0; i < channelCount; ++i) {
Chris@363 567 writebuf[i] = buffers[i] + offset;
Chris@363 568 }
Chris@363 569 }
Chris@363 570
Chris@363 571 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
Chris@363 572
Chris@363 573 if (writebuf != buffers) delete[] writebuf;
Chris@363 574 }
Chris@320 575
Chris@320 576 while (got < size) {
Chris@363 577 for (int c = 0; c < channelCount; ++c) {
Chris@363 578 buffers[c][got + offset] = 0.0;
Chris@363 579 }
Chris@320 580 ++got;
Chris@320 581 }
Chris@320 582 }
Chris@320 583
Chris@320 584 void
Chris@331 585 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
Chris@320 586 const Vamp::Plugin::Feature &feature)
Chris@320 587 {
Chris@350 588 size_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 589
Chris@331 590 // std::cerr << "FeatureExtractionModelTransformer::addFeature("
Chris@320 591 // << blockFrame << ")" << std::endl;
Chris@320 592
Chris@320 593 int binCount = 1;
Chris@320 594 if (m_descriptor->hasFixedBinCount) {
Chris@320 595 binCount = m_descriptor->binCount;
Chris@320 596 }
Chris@320 597
Chris@320 598 size_t frame = blockFrame;
Chris@320 599
Chris@320 600 if (m_descriptor->sampleType ==
Chris@320 601 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 602
Chris@320 603 if (!feature.hasTimestamp) {
Chris@320 604 std::cerr
Chris@331 605 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@320 606 << "Feature has variable sample rate but no timestamp!"
Chris@320 607 << std::endl;
Chris@320 608 return;
Chris@320 609 } else {
Chris@320 610 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@320 611 }
Chris@320 612
Chris@320 613 } else if (m_descriptor->sampleType ==
Chris@320 614 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 615
Chris@320 616 if (feature.hasTimestamp) {
Chris@320 617 //!!! warning: sampleRate may be non-integral
Chris@320 618 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
Chris@451 619 //!!! see comment above when setting up modelResolution and modelRate
Chris@451 620 // lrintf(m_descriptor->sampleRate));
Chris@451 621 inputRate);
Chris@320 622 } else {
Chris@320 623 frame = m_output->getEndFrame();
Chris@320 624 }
Chris@320 625 }
Chris@320 626
Chris@441 627 // Rather than repeat the complicated tests from the constructor
Chris@441 628 // to determine what sort of model we must be adding the features
Chris@441 629 // to, we instead test what sort of model the constructor decided
Chris@441 630 // to create.
Chris@320 631
Chris@441 632 if (isOutput<SparseOneDimensionalModel>()) {
Chris@441 633
Chris@441 634 SparseOneDimensionalModel *model =
Chris@350 635 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 636 if (!model) return;
Chris@350 637
Chris@441 638 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 639 (frame, feature.label.c_str()));
Chris@320 640
Chris@441 641 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 642
Chris@350 643 SparseTimeValueModel *model =
Chris@350 644 getConformingOutput<SparseTimeValueModel>();
Chris@320 645 if (!model) return;
Chris@350 646
Chris@454 647 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 648
Chris@454 649 float value = feature.values[i];
Chris@454 650
Chris@454 651 QString label = feature.label.c_str();
Chris@454 652 if (feature.values.size() > 1) {
Chris@454 653 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 654 }
Chris@454 655
Chris@454 656 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
Chris@454 657 }
Chris@320 658
Chris@441 659 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
Chris@320 660
Chris@441 661 int index = 0;
Chris@441 662
Chris@441 663 float value = 0.0;
Chris@441 664 if (feature.values.size() > index) {
Chris@441 665 value = feature.values[index++];
Chris@441 666 }
Chris@320 667
Chris@320 668 float duration = 1;
Chris@441 669 if (feature.hasDuration) {
Chris@441 670 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 671 } else {
Chris@441 672 if (feature.values.size() > index) {
Chris@441 673 duration = feature.values[index++];
Chris@441 674 }
Chris@441 675 }
Chris@320 676
Chris@441 677 if (isOutput<NoteModel>()) {
Chris@320 678
Chris@441 679 float velocity = 100;
Chris@441 680 if (feature.values.size() > index) {
Chris@441 681 velocity = feature.values[index++];
Chris@441 682 }
Chris@441 683 if (velocity < 0) velocity = 127;
Chris@441 684 if (velocity > 127) velocity = 127;
Chris@320 685
Chris@441 686 NoteModel *model = getConformingOutput<NoteModel>();
Chris@441 687 if (!model) return;
Chris@441 688 model->addPoint(NoteModel::Point(frame, value, // value is pitch
Chris@441 689 lrintf(duration),
Chris@441 690 velocity / 127.f,
Chris@441 691 feature.label.c_str()));
Chris@441 692 } else {
Chris@441 693 RegionModel *model = getConformingOutput<RegionModel>();
Chris@454 694 if (!model) return;
Chris@454 695
Chris@474 696 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 697
Chris@454 698 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 699
Chris@454 700 float value = feature.values[i];
Chris@454 701
Chris@454 702 QString label = feature.label.c_str();
Chris@454 703 if (feature.values.size() > 1) {
Chris@454 704 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 705 }
Chris@454 706
Chris@454 707 model->addPoint(RegionModel::Point(frame, value,
Chris@454 708 lrintf(duration),
Chris@454 709 label));
Chris@454 710 }
Chris@454 711 } else {
Chris@454 712
Chris@441 713 model->addPoint(RegionModel::Point(frame, value,
Chris@441 714 lrintf(duration),
Chris@441 715 feature.label.c_str()));
Chris@454 716 }
Chris@441 717 }
Chris@320 718
Chris@441 719 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 720
Chris@533 721 DenseThreeDimensionalModel::Column values =
Chris@533 722 DenseThreeDimensionalModel::Column::fromStdVector(feature.values);
Chris@320 723
Chris@320 724 EditableDenseThreeDimensionalModel *model =
Chris@350 725 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 726 if (!model) return;
Chris@320 727
Chris@320 728 model->setColumn(frame / model->getResolution(), values);
Chris@441 729
Chris@441 730 } else {
Chris@441 731 std::cerr << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << std::endl;
Chris@320 732 }
Chris@320 733 }
Chris@320 734
Chris@320 735 void
Chris@331 736 FeatureExtractionModelTransformer::setCompletion(int completion)
Chris@320 737 {
Chris@320 738 int binCount = 1;
Chris@320 739 if (m_descriptor->hasFixedBinCount) {
Chris@320 740 binCount = m_descriptor->binCount;
Chris@320 741 }
Chris@320 742
Chris@331 743 // std::cerr << "FeatureExtractionModelTransformer::setCompletion("
Chris@320 744 // << completion << ")" << std::endl;
Chris@320 745
Chris@441 746 if (isOutput<SparseOneDimensionalModel>()) {
Chris@320 747
Chris@350 748 SparseOneDimensionalModel *model =
Chris@350 749 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 750 if (!model) return;
Chris@441 751 model->setCompletion(completion, true);
Chris@320 752
Chris@441 753 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 754
Chris@350 755 SparseTimeValueModel *model =
Chris@350 756 getConformingOutput<SparseTimeValueModel>();
Chris@320 757 if (!model) return;
Chris@441 758 model->setCompletion(completion, true);
Chris@320 759
Chris@441 760 } else if (isOutput<NoteModel>()) {
Chris@320 761
Chris@441 762 NoteModel *model = getConformingOutput<NoteModel>();
Chris@320 763 if (!model) return;
Chris@441 764 model->setCompletion(completion, true);
Chris@320 765
Chris@441 766 } else if (isOutput<RegionModel>()) {
Chris@441 767
Chris@441 768 RegionModel *model = getConformingOutput<RegionModel>();
Chris@441 769 if (!model) return;
Chris@441 770 model->setCompletion(completion, true);
Chris@441 771
Chris@441 772 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 773
Chris@320 774 EditableDenseThreeDimensionalModel *model =
Chris@350 775 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 776 if (!model) return;
Chris@350 777 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 778 }
Chris@320 779 }
Chris@320 780