annotate transform/FeatureExtractionModelTransformer.cpp @ 489:82ab61fa9223

* Reorganise our sparql queries on the basis that Redland must be available, not only optional. So for anything querying the pool of data about plugins, we use a single datastore and model which is initialised at the outset by PluginRDFIndexer and then queried directly; for anything that "reads from a file" (e.g. loading annotations) we query directly using Rasqal, going to the datastore when we need additional plugin-related information. This may improve performance, but mostly it simplifies the code and fixes a serious issue with RDF import in the previous versions (namely that multiple sequential RDF imports would end up sharing the same RDF data pool!)
author Chris Cannam
date Fri, 21 Nov 2008 16:12:29 +0000
parents a70dcfed59c1
children b6dc6c7f402c
rev   line source
Chris@320 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@320 2
Chris@320 3 /*
Chris@320 4 Sonic Visualiser
Chris@320 5 An audio file viewer and annotation editor.
Chris@320 6 Centre for Digital Music, Queen Mary, University of London.
Chris@320 7 This file copyright 2006 Chris Cannam and QMUL.
Chris@320 8
Chris@320 9 This program is free software; you can redistribute it and/or
Chris@320 10 modify it under the terms of the GNU General Public License as
Chris@320 11 published by the Free Software Foundation; either version 2 of the
Chris@320 12 License, or (at your option) any later version. See the file
Chris@320 13 COPYING included with this distribution for more information.
Chris@320 14 */
Chris@320 15
Chris@331 16 #include "FeatureExtractionModelTransformer.h"
Chris@320 17
Chris@320 18 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@320 19 #include "plugin/PluginXml.h"
Chris@475 20 #include <vamp-hostsdk/Plugin.h>
Chris@320 21
Chris@320 22 #include "data/model/Model.h"
Chris@320 23 #include "base/Window.h"
Chris@387 24 #include "base/Exceptions.h"
Chris@320 25 #include "data/model/SparseOneDimensionalModel.h"
Chris@320 26 #include "data/model/SparseTimeValueModel.h"
Chris@320 27 #include "data/model/EditableDenseThreeDimensionalModel.h"
Chris@320 28 #include "data/model/DenseTimeValueModel.h"
Chris@320 29 #include "data/model/NoteModel.h"
Chris@441 30 #include "data/model/RegionModel.h"
Chris@320 31 #include "data/model/FFTModel.h"
Chris@320 32 #include "data/model/WaveFileModel.h"
Chris@320 33
Chris@350 34 #include "TransformFactory.h"
Chris@350 35
Chris@320 36 #include <iostream>
Chris@320 37
Chris@350 38 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
Chris@350 39 const Transform &transform) :
Chris@350 40 ModelTransformer(in, transform),
Chris@320 41 m_plugin(0),
Chris@320 42 m_descriptor(0),
Chris@320 43 m_outputFeatureNo(0)
Chris@320 44 {
Chris@350 45 // std::cerr << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId.toStdString() << ", outputName " << m_transform.getOutput().toStdString() << std::endl;
Chris@350 46
Chris@350 47 QString pluginId = transform.getPluginIdentifier();
Chris@320 48
Chris@320 49 FeatureExtractionPluginFactory *factory =
Chris@320 50 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@320 51
Chris@320 52 if (!factory) {
Chris@361 53 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
Chris@320 54 return;
Chris@320 55 }
Chris@320 56
Chris@350 57 DenseTimeValueModel *input = getConformingInput();
Chris@350 58 if (!input) {
Chris@361 59 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
Chris@350 60 return;
Chris@350 61 }
Chris@320 62
Chris@350 63 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
Chris@320 64 if (!m_plugin) {
Chris@361 65 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
Chris@320 66 return;
Chris@320 67 }
Chris@320 68
Chris@350 69 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@350 70 (m_transform, m_plugin);
Chris@343 71
Chris@350 72 TransformFactory::getInstance()->setPluginParameters
Chris@350 73 (m_transform, m_plugin);
Chris@320 74
Chris@320 75 size_t channelCount = input->getChannelCount();
Chris@320 76 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 77 channelCount = 1;
Chris@320 78 }
Chris@320 79 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@361 80 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
Chris@361 81 .arg(pluginId)
Chris@361 82 .arg(m_plugin->getMinChannelCount())
Chris@361 83 .arg(m_plugin->getMaxChannelCount())
Chris@361 84 .arg(input->getChannelCount());
Chris@320 85 return;
Chris@320 86 }
Chris@320 87
Chris@320 88 std::cerr << "Initialising feature extraction plugin with channels = "
Chris@350 89 << channelCount << ", step = " << m_transform.getStepSize()
Chris@350 90 << ", block = " << m_transform.getBlockSize() << std::endl;
Chris@320 91
Chris@320 92 if (!m_plugin->initialise(channelCount,
Chris@350 93 m_transform.getStepSize(),
Chris@350 94 m_transform.getBlockSize())) {
Chris@361 95
Chris@361 96 size_t pstep = m_transform.getStepSize();
Chris@361 97 size_t pblock = m_transform.getBlockSize();
Chris@361 98
Chris@361 99 m_transform.setStepSize(0);
Chris@361 100 m_transform.setBlockSize(0);
Chris@361 101 TransformFactory::getInstance()->makeContextConsistentWithPlugin
Chris@361 102 (m_transform, m_plugin);
Chris@361 103
Chris@361 104 if (m_transform.getStepSize() != pstep ||
Chris@361 105 m_transform.getBlockSize() != pblock) {
Chris@361 106
Chris@361 107 if (!m_plugin->initialise(channelCount,
Chris@361 108 m_transform.getStepSize(),
Chris@361 109 m_transform.getBlockSize())) {
Chris@361 110
Chris@361 111 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 112 return;
Chris@361 113
Chris@361 114 } else {
Chris@361 115
Chris@361 116 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
Chris@361 117 .arg(pluginId)
Chris@361 118 .arg(pstep)
Chris@361 119 .arg(pblock)
Chris@361 120 .arg(m_transform.getStepSize())
Chris@361 121 .arg(m_transform.getBlockSize());
Chris@361 122 }
Chris@361 123
Chris@361 124 } else {
Chris@361 125
Chris@361 126 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
Chris@361 127 return;
Chris@361 128 }
Chris@320 129 }
Chris@320 130
Chris@366 131 if (m_transform.getPluginVersion() != "") {
Chris@366 132 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
Chris@366 133 if (pv != m_transform.getPluginVersion()) {
Chris@366 134 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
Chris@366 135 .arg(m_transform.getPluginVersion())
Chris@366 136 .arg(pluginId)
Chris@366 137 .arg(pv);
Chris@366 138 if (m_message != "") {
Chris@366 139 m_message = QString("%1; %2").arg(vm).arg(m_message);
Chris@366 140 } else {
Chris@366 141 m_message = vm;
Chris@366 142 }
Chris@366 143 }
Chris@366 144 }
Chris@366 145
Chris@320 146 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
Chris@320 147
Chris@320 148 if (outputs.empty()) {
Chris@361 149 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
Chris@320 150 return;
Chris@320 151 }
Chris@320 152
Chris@320 153 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@429 154 // std::cerr << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput().toStdString() << "\"" << std::endl;
Chris@350 155 if (m_transform.getOutput() == "" ||
Chris@350 156 outputs[i].identifier == m_transform.getOutput().toStdString()) {
Chris@320 157 m_outputFeatureNo = i;
Chris@441 158 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
Chris@320 159 break;
Chris@320 160 }
Chris@320 161 }
Chris@320 162
Chris@320 163 if (!m_descriptor) {
Chris@361 164 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
Chris@361 165 .arg(pluginId)
Chris@361 166 .arg(m_transform.getOutput());
Chris@320 167 return;
Chris@320 168 }
Chris@320 169
Chris@331 170 // std::cerr << "FeatureExtractionModelTransformer: output sample type "
Chris@320 171 // << m_descriptor->sampleType << std::endl;
Chris@320 172
Chris@320 173 int binCount = 1;
Chris@320 174 float minValue = 0.0, maxValue = 0.0;
Chris@320 175 bool haveExtents = false;
Chris@320 176
Chris@320 177 if (m_descriptor->hasFixedBinCount) {
Chris@320 178 binCount = m_descriptor->binCount;
Chris@320 179 }
Chris@320 180
Chris@331 181 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
Chris@320 182 // << binCount << std::endl;
Chris@320 183
Chris@320 184 if (binCount > 0 && m_descriptor->hasKnownExtents) {
Chris@320 185 minValue = m_descriptor->minValue;
Chris@320 186 maxValue = m_descriptor->maxValue;
Chris@320 187 haveExtents = true;
Chris@320 188 }
Chris@320 189
Chris@350 190 size_t modelRate = input->getSampleRate();
Chris@320 191 size_t modelResolution = 1;
Chris@320 192
Chris@320 193 switch (m_descriptor->sampleType) {
Chris@320 194
Chris@320 195 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
Chris@320 196 if (m_descriptor->sampleRate != 0.0) {
Chris@320 197 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@320 198 }
Chris@320 199 break;
Chris@320 200
Chris@320 201 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
Chris@350 202 modelResolution = m_transform.getStepSize();
Chris@320 203 break;
Chris@320 204
Chris@320 205 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
Chris@451 206 //!!! SV doesn't actually support display of models that have
Chris@451 207 //!!! different underlying rates together -- so we always set
Chris@451 208 //!!! the model rate to be the input model's rate, and adjust
Chris@451 209 //!!! the resolution appropriately. We can't properly display
Chris@451 210 //!!! data with a higher resolution than the base model at all
Chris@451 211 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
Chris@451 212 if (m_descriptor->sampleRate > input->getSampleRate()) {
Chris@451 213 modelResolution = 1;
Chris@451 214 } else {
Chris@451 215 modelResolution = size_t(input->getSampleRate() /
Chris@451 216 m_descriptor->sampleRate);
Chris@451 217 }
Chris@320 218 break;
Chris@320 219 }
Chris@320 220
Chris@441 221 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
Chris@441 222
Chris@441 223 if (binCount == 0 &&
Chris@441 224 (preDurationPlugin || !m_descriptor->hasDuration)) {
Chris@320 225
Chris@445 226 // Anything with no value and no duration is an instant
Chris@445 227
Chris@320 228 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
Chris@320 229 false);
Chris@320 230
Chris@441 231 } else if ((preDurationPlugin && binCount > 1 &&
Chris@441 232 (m_descriptor->sampleType ==
Chris@441 233 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
Chris@441 234 (!preDurationPlugin && m_descriptor->hasDuration)) {
Chris@441 235
Chris@441 236 // For plugins using the old v1 API without explicit duration,
Chris@441 237 // we treat anything that has multiple bins (i.e. that has the
Chris@441 238 // potential to have value and duration) and a variable sample
Chris@441 239 // rate as a note model, taking its values as pitch, duration
Chris@441 240 // and velocity (if present) respectively. This is the same
Chris@441 241 // behaviour as always applied by SV to these plugins in the
Chris@441 242 // past.
Chris@441 243
Chris@441 244 // For plugins with the newer API, we treat anything with
Chris@441 245 // duration as either a note model with pitch and velocity, or
Chris@441 246 // a region model.
Chris@441 247
Chris@441 248 // How do we know whether it's an interval or note model?
Chris@441 249 // What's the essential difference? Is a note model any
Chris@441 250 // interval model using a Hz or "MIDI pitch" scale? There
Chris@441 251 // isn't really a reliable test for "MIDI pitch"... Does a
Chris@441 252 // note model always have velocity? This is a good question
Chris@441 253 // to be addressed by accompanying RDF, but for the moment we
Chris@441 254 // will do the following...
Chris@441 255
Chris@441 256 bool isNoteModel = false;
Chris@441 257
Chris@441 258 // Regions have only value (and duration -- we can't extract a
Chris@441 259 // region model from an old-style plugin that doesn't support
Chris@441 260 // duration)
Chris@441 261 if (binCount > 1) isNoteModel = true;
Chris@441 262
Chris@441 263 // Regions do not have units of Hz (a sweeping assumption!)
Chris@441 264 if (m_descriptor->unit == "Hz") isNoteModel = true;
Chris@441 265
Chris@441 266 // If we had a "sparse 3D model", we would have the additional
Chris@441 267 // problem of determining whether to use that here (if bin
Chris@441 268 // count > 1). But we don't.
Chris@441 269
Chris@441 270 if (isNoteModel) {
Chris@441 271
Chris@441 272 NoteModel *model;
Chris@441 273 if (haveExtents) {
Chris@441 274 model = new NoteModel
Chris@441 275 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 276 } else {
Chris@441 277 model = new NoteModel
Chris@441 278 (modelRate, modelResolution, false);
Chris@441 279 }
Chris@441 280 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 281 m_output = model;
Chris@441 282
Chris@441 283 } else {
Chris@441 284
Chris@441 285 RegionModel *model;
Chris@441 286 if (haveExtents) {
Chris@441 287 model = new RegionModel
Chris@441 288 (modelRate, modelResolution, minValue, maxValue, false);
Chris@441 289 } else {
Chris@441 290 model = new RegionModel
Chris@441 291 (modelRate, modelResolution, false);
Chris@441 292 }
Chris@441 293 model->setScaleUnits(m_descriptor->unit.c_str());
Chris@441 294 m_output = model;
Chris@441 295 }
Chris@441 296
Chris@441 297 } else if (binCount == 1 ||
Chris@441 298 (m_descriptor->sampleType ==
Chris@441 299 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
Chris@441 300
Chris@441 301 // Anything that is not a 1D, note, or interval model and that
Chris@441 302 // has only one value per result must be a sparse time value
Chris@441 303 // model.
Chris@441 304
Chris@441 305 // Anything that is not a 1D, note, or interval model and that
Chris@441 306 // has a variable sample rate is also treated as a sparse time
Chris@441 307 // value model regardless of its bin count, because we lack a
Chris@441 308 // sparse 3D model.
Chris@320 309
Chris@320 310 SparseTimeValueModel *model;
Chris@320 311 if (haveExtents) {
Chris@320 312 model = new SparseTimeValueModel
Chris@320 313 (modelRate, modelResolution, minValue, maxValue, false);
Chris@320 314 } else {
Chris@320 315 model = new SparseTimeValueModel
Chris@320 316 (modelRate, modelResolution, false);
Chris@320 317 }
Chris@320 318 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
Chris@320 319
Chris@320 320 m_output = model;
Chris@320 321
Chris@441 322 } else {
Chris@320 323
Chris@441 324 // Anything that is not a 1D, note, or interval model and that
Chris@441 325 // has a fixed sample rate and more than one value per result
Chris@441 326 // must be a dense 3D model.
Chris@320 327
Chris@320 328 EditableDenseThreeDimensionalModel *model =
Chris@320 329 new EditableDenseThreeDimensionalModel
Chris@320 330 (modelRate, modelResolution, binCount, false);
Chris@320 331
Chris@320 332 if (!m_descriptor->binNames.empty()) {
Chris@320 333 std::vector<QString> names;
Chris@320 334 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
Chris@320 335 names.push_back(m_descriptor->binNames[i].c_str());
Chris@320 336 }
Chris@320 337 model->setBinNames(names);
Chris@320 338 }
Chris@320 339
Chris@320 340 m_output = model;
Chris@320 341 }
Chris@333 342
Chris@350 343 if (m_output) m_output->setSourceModel(input);
Chris@320 344 }
Chris@320 345
Chris@331 346 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
Chris@320 347 {
Chris@436 348 // std::cerr << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << std::endl;
Chris@320 349 delete m_plugin;
Chris@320 350 delete m_descriptor;
Chris@320 351 }
Chris@320 352
Chris@320 353 DenseTimeValueModel *
Chris@350 354 FeatureExtractionModelTransformer::getConformingInput()
Chris@320 355 {
Chris@408 356 // std::cerr << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << std::endl;
Chris@408 357
Chris@320 358 DenseTimeValueModel *dtvm =
Chris@320 359 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@320 360 if (!dtvm) {
Chris@350 361 std::cerr << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
Chris@320 362 }
Chris@320 363 return dtvm;
Chris@320 364 }
Chris@320 365
Chris@320 366 void
Chris@331 367 FeatureExtractionModelTransformer::run()
Chris@320 368 {
Chris@350 369 DenseTimeValueModel *input = getConformingInput();
Chris@320 370 if (!input) return;
Chris@320 371
Chris@320 372 if (!m_output) return;
Chris@320 373
Chris@320 374 while (!input->isReady()) {
Chris@331 375 std::cerr << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << std::endl;
Chris@320 376 sleep(1);
Chris@320 377 }
Chris@320 378
Chris@350 379 size_t sampleRate = input->getSampleRate();
Chris@320 380
Chris@320 381 size_t channelCount = input->getChannelCount();
Chris@320 382 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@320 383 channelCount = 1;
Chris@320 384 }
Chris@320 385
Chris@320 386 float **buffers = new float*[channelCount];
Chris@320 387 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 388 buffers[ch] = new float[m_transform.getBlockSize() + 2];
Chris@320 389 }
Chris@320 390
Chris@350 391 size_t stepSize = m_transform.getStepSize();
Chris@350 392 size_t blockSize = m_transform.getBlockSize();
Chris@350 393
Chris@320 394 bool frequencyDomain = (m_plugin->getInputDomain() ==
Chris@320 395 Vamp::Plugin::FrequencyDomain);
Chris@320 396 std::vector<FFTModel *> fftModels;
Chris@320 397
Chris@320 398 if (frequencyDomain) {
Chris@320 399 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 400 FFTModel *model = new FFTModel
Chris@350 401 (getConformingInput(),
Chris@350 402 channelCount == 1 ? m_input.getChannel() : ch,
Chris@350 403 m_transform.getWindowType(),
Chris@350 404 blockSize,
Chris@350 405 stepSize,
Chris@350 406 blockSize,
Chris@334 407 false,
Chris@334 408 StorageAdviser::PrecisionCritical);
Chris@320 409 if (!model->isOK()) {
Chris@320 410 delete model;
Chris@320 411 setCompletion(100);
Chris@387 412 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
Chris@387 413 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
Chris@320 414 }
Chris@320 415 model->resume();
Chris@320 416 fftModels.push_back(model);
Chris@320 417 }
Chris@320 418 }
Chris@320 419
Chris@350 420 long startFrame = m_input.getModel()->getStartFrame();
Chris@350 421 long endFrame = m_input.getModel()->getEndFrame();
Chris@320 422
Chris@350 423 RealTime contextStartRT = m_transform.getStartTime();
Chris@350 424 RealTime contextDurationRT = m_transform.getDuration();
Chris@350 425
Chris@350 426 long contextStart =
Chris@350 427 RealTime::realTime2Frame(contextStartRT, sampleRate);
Chris@350 428
Chris@350 429 long contextDuration =
Chris@350 430 RealTime::realTime2Frame(contextDurationRT, sampleRate);
Chris@320 431
Chris@320 432 if (contextStart == 0 || contextStart < startFrame) {
Chris@320 433 contextStart = startFrame;
Chris@320 434 }
Chris@320 435
Chris@320 436 if (contextDuration == 0) {
Chris@320 437 contextDuration = endFrame - contextStart;
Chris@320 438 }
Chris@320 439 if (contextStart + contextDuration > endFrame) {
Chris@320 440 contextDuration = endFrame - contextStart;
Chris@320 441 }
Chris@320 442
Chris@320 443 long blockFrame = contextStart;
Chris@320 444
Chris@320 445 long prevCompletion = 0;
Chris@320 446
Chris@320 447 setCompletion(0);
Chris@320 448
Chris@320 449 while (!m_abandoned) {
Chris@320 450
Chris@320 451 if (frequencyDomain) {
Chris@350 452 if (blockFrame - int(blockSize)/2 >
Chris@320 453 contextStart + contextDuration) break;
Chris@320 454 } else {
Chris@320 455 if (blockFrame >=
Chris@320 456 contextStart + contextDuration) break;
Chris@320 457 }
Chris@320 458
Chris@331 459 // std::cerr << "FeatureExtractionModelTransformer::run: blockFrame "
Chris@320 460 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
Chris@350 461 // << blockSize << std::endl;
Chris@320 462
Chris@320 463 long completion =
Chris@350 464 (((blockFrame - contextStart) / stepSize) * 99) /
Chris@350 465 (contextDuration / stepSize);
Chris@320 466
Chris@350 467 // channelCount is either m_input.getModel()->channelCount or 1
Chris@320 468
Chris@363 469 if (frequencyDomain) {
Chris@363 470 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@350 471 int column = (blockFrame - startFrame) / stepSize;
Chris@350 472 for (size_t i = 0; i <= blockSize/2; ++i) {
Chris@320 473 fftModels[ch]->getValuesAt
Chris@320 474 (column, i, buffers[ch][i*2], buffers[ch][i*2+1]);
Chris@320 475 }
Chris@363 476 }
Chris@363 477 } else {
Chris@363 478 getFrames(channelCount, blockFrame, blockSize, buffers);
Chris@320 479 }
Chris@320 480
Chris@320 481 Vamp::Plugin::FeatureSet features = m_plugin->process
Chris@320 482 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@320 483
Chris@320 484 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@320 485 Vamp::Plugin::Feature feature =
Chris@320 486 features[m_outputFeatureNo][fi];
Chris@320 487 addFeature(blockFrame, feature);
Chris@320 488 }
Chris@320 489
Chris@320 490 if (blockFrame == contextStart || completion > prevCompletion) {
Chris@320 491 setCompletion(completion);
Chris@320 492 prevCompletion = completion;
Chris@320 493 }
Chris@320 494
Chris@350 495 blockFrame += stepSize;
Chris@320 496 }
Chris@320 497
Chris@320 498 if (m_abandoned) return;
Chris@320 499
Chris@320 500 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@320 501
Chris@320 502 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@320 503 Vamp::Plugin::Feature feature =
Chris@320 504 features[m_outputFeatureNo][fi];
Chris@320 505 addFeature(blockFrame, feature);
Chris@320 506 }
Chris@320 507
Chris@320 508 if (frequencyDomain) {
Chris@320 509 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@320 510 delete fftModels[ch];
Chris@320 511 }
Chris@320 512 }
Chris@320 513
Chris@320 514 setCompletion(100);
Chris@320 515 }
Chris@320 516
Chris@320 517 void
Chris@363 518 FeatureExtractionModelTransformer::getFrames(int channelCount,
Chris@363 519 long startFrame, long size,
Chris@363 520 float **buffers)
Chris@320 521 {
Chris@320 522 long offset = 0;
Chris@320 523
Chris@320 524 if (startFrame < 0) {
Chris@363 525 for (int c = 0; c < channelCount; ++c) {
Chris@363 526 for (int i = 0; i < size && startFrame + i < 0; ++i) {
Chris@363 527 buffers[c][i] = 0.0f;
Chris@363 528 }
Chris@320 529 }
Chris@320 530 offset = -startFrame;
Chris@320 531 size -= offset;
Chris@320 532 if (size <= 0) return;
Chris@320 533 startFrame = 0;
Chris@320 534 }
Chris@320 535
Chris@350 536 DenseTimeValueModel *input = getConformingInput();
Chris@350 537 if (!input) return;
Chris@363 538
Chris@363 539 long got = 0;
Chris@350 540
Chris@363 541 if (channelCount == 1) {
Chris@363 542
Chris@363 543 got = input->getData(m_input.getChannel(), startFrame, size,
Chris@363 544 buffers[0] + offset);
Chris@363 545
Chris@363 546 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
Chris@363 547 // use mean instead of sum, as plugin input
Chris@363 548 float cc = float(input->getChannelCount());
Chris@363 549 for (long i = 0; i < size; ++i) {
Chris@363 550 buffers[0][i + offset] /= cc;
Chris@363 551 }
Chris@363 552 }
Chris@363 553
Chris@363 554 } else {
Chris@363 555
Chris@363 556 float **writebuf = buffers;
Chris@363 557 if (offset > 0) {
Chris@363 558 writebuf = new float *[channelCount];
Chris@363 559 for (int i = 0; i < channelCount; ++i) {
Chris@363 560 writebuf[i] = buffers[i] + offset;
Chris@363 561 }
Chris@363 562 }
Chris@363 563
Chris@363 564 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
Chris@363 565
Chris@363 566 if (writebuf != buffers) delete[] writebuf;
Chris@363 567 }
Chris@320 568
Chris@320 569 while (got < size) {
Chris@363 570 for (int c = 0; c < channelCount; ++c) {
Chris@363 571 buffers[c][got + offset] = 0.0;
Chris@363 572 }
Chris@320 573 ++got;
Chris@320 574 }
Chris@320 575 }
Chris@320 576
Chris@320 577 void
Chris@331 578 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
Chris@320 579 const Vamp::Plugin::Feature &feature)
Chris@320 580 {
Chris@350 581 size_t inputRate = m_input.getModel()->getSampleRate();
Chris@320 582
Chris@331 583 // std::cerr << "FeatureExtractionModelTransformer::addFeature("
Chris@320 584 // << blockFrame << ")" << std::endl;
Chris@320 585
Chris@320 586 int binCount = 1;
Chris@320 587 if (m_descriptor->hasFixedBinCount) {
Chris@320 588 binCount = m_descriptor->binCount;
Chris@320 589 }
Chris@320 590
Chris@320 591 size_t frame = blockFrame;
Chris@320 592
Chris@320 593 if (m_descriptor->sampleType ==
Chris@320 594 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
Chris@320 595
Chris@320 596 if (!feature.hasTimestamp) {
Chris@320 597 std::cerr
Chris@331 598 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
Chris@320 599 << "Feature has variable sample rate but no timestamp!"
Chris@320 600 << std::endl;
Chris@320 601 return;
Chris@320 602 } else {
Chris@320 603 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@320 604 }
Chris@320 605
Chris@320 606 } else if (m_descriptor->sampleType ==
Chris@320 607 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
Chris@320 608
Chris@320 609 if (feature.hasTimestamp) {
Chris@320 610 //!!! warning: sampleRate may be non-integral
Chris@320 611 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
Chris@451 612 //!!! see comment above when setting up modelResolution and modelRate
Chris@451 613 // lrintf(m_descriptor->sampleRate));
Chris@451 614 inputRate);
Chris@320 615 } else {
Chris@320 616 frame = m_output->getEndFrame();
Chris@320 617 }
Chris@320 618 }
Chris@320 619
Chris@441 620 // Rather than repeat the complicated tests from the constructor
Chris@441 621 // to determine what sort of model we must be adding the features
Chris@441 622 // to, we instead test what sort of model the constructor decided
Chris@441 623 // to create.
Chris@320 624
Chris@441 625 if (isOutput<SparseOneDimensionalModel>()) {
Chris@441 626
Chris@441 627 SparseOneDimensionalModel *model =
Chris@350 628 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 629 if (!model) return;
Chris@350 630
Chris@441 631 model->addPoint(SparseOneDimensionalModel::Point
Chris@441 632 (frame, feature.label.c_str()));
Chris@320 633
Chris@441 634 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 635
Chris@350 636 SparseTimeValueModel *model =
Chris@350 637 getConformingOutput<SparseTimeValueModel>();
Chris@320 638 if (!model) return;
Chris@350 639
Chris@454 640 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 641
Chris@454 642 float value = feature.values[i];
Chris@454 643
Chris@454 644 QString label = feature.label.c_str();
Chris@454 645 if (feature.values.size() > 1) {
Chris@454 646 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 647 }
Chris@454 648
Chris@454 649 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
Chris@454 650 }
Chris@320 651
Chris@441 652 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
Chris@320 653
Chris@441 654 int index = 0;
Chris@441 655
Chris@441 656 float value = 0.0;
Chris@441 657 if (feature.values.size() > index) {
Chris@441 658 value = feature.values[index++];
Chris@441 659 }
Chris@320 660
Chris@320 661 float duration = 1;
Chris@441 662 if (feature.hasDuration) {
Chris@441 663 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
Chris@441 664 } else {
Chris@441 665 if (feature.values.size() > index) {
Chris@441 666 duration = feature.values[index++];
Chris@441 667 }
Chris@441 668 }
Chris@320 669
Chris@441 670 if (isOutput<NoteModel>()) {
Chris@320 671
Chris@441 672 float velocity = 100;
Chris@441 673 if (feature.values.size() > index) {
Chris@441 674 velocity = feature.values[index++];
Chris@441 675 }
Chris@441 676 if (velocity < 0) velocity = 127;
Chris@441 677 if (velocity > 127) velocity = 127;
Chris@320 678
Chris@441 679 NoteModel *model = getConformingOutput<NoteModel>();
Chris@441 680 if (!model) return;
Chris@441 681 model->addPoint(NoteModel::Point(frame, value, // value is pitch
Chris@441 682 lrintf(duration),
Chris@441 683 velocity / 127.f,
Chris@441 684 feature.label.c_str()));
Chris@441 685 } else {
Chris@441 686 RegionModel *model = getConformingOutput<RegionModel>();
Chris@454 687 if (!model) return;
Chris@454 688
Chris@474 689 if (feature.hasDuration && !feature.values.empty()) {
Chris@454 690
Chris@454 691 for (int i = 0; i < feature.values.size(); ++i) {
Chris@454 692
Chris@454 693 float value = feature.values[i];
Chris@454 694
Chris@454 695 QString label = feature.label.c_str();
Chris@454 696 if (feature.values.size() > 1) {
Chris@454 697 label = QString("[%1] %2").arg(i+1).arg(label);
Chris@454 698 }
Chris@454 699
Chris@454 700 model->addPoint(RegionModel::Point(frame, value,
Chris@454 701 lrintf(duration),
Chris@454 702 label));
Chris@454 703 }
Chris@454 704 } else {
Chris@454 705
Chris@441 706 model->addPoint(RegionModel::Point(frame, value,
Chris@441 707 lrintf(duration),
Chris@441 708 feature.label.c_str()));
Chris@454 709 }
Chris@441 710 }
Chris@320 711
Chris@441 712 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 713
Chris@320 714 DenseThreeDimensionalModel::Column values = feature.values;
Chris@320 715
Chris@320 716 EditableDenseThreeDimensionalModel *model =
Chris@350 717 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 718 if (!model) return;
Chris@320 719
Chris@320 720 model->setColumn(frame / model->getResolution(), values);
Chris@441 721
Chris@441 722 } else {
Chris@441 723 std::cerr << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << std::endl;
Chris@320 724 }
Chris@320 725 }
Chris@320 726
Chris@320 727 void
Chris@331 728 FeatureExtractionModelTransformer::setCompletion(int completion)
Chris@320 729 {
Chris@320 730 int binCount = 1;
Chris@320 731 if (m_descriptor->hasFixedBinCount) {
Chris@320 732 binCount = m_descriptor->binCount;
Chris@320 733 }
Chris@320 734
Chris@331 735 // std::cerr << "FeatureExtractionModelTransformer::setCompletion("
Chris@320 736 // << completion << ")" << std::endl;
Chris@320 737
Chris@441 738 if (isOutput<SparseOneDimensionalModel>()) {
Chris@320 739
Chris@350 740 SparseOneDimensionalModel *model =
Chris@350 741 getConformingOutput<SparseOneDimensionalModel>();
Chris@320 742 if (!model) return;
Chris@441 743 model->setCompletion(completion, true);
Chris@320 744
Chris@441 745 } else if (isOutput<SparseTimeValueModel>()) {
Chris@320 746
Chris@350 747 SparseTimeValueModel *model =
Chris@350 748 getConformingOutput<SparseTimeValueModel>();
Chris@320 749 if (!model) return;
Chris@441 750 model->setCompletion(completion, true);
Chris@320 751
Chris@441 752 } else if (isOutput<NoteModel>()) {
Chris@320 753
Chris@441 754 NoteModel *model = getConformingOutput<NoteModel>();
Chris@320 755 if (!model) return;
Chris@441 756 model->setCompletion(completion, true);
Chris@320 757
Chris@441 758 } else if (isOutput<RegionModel>()) {
Chris@441 759
Chris@441 760 RegionModel *model = getConformingOutput<RegionModel>();
Chris@441 761 if (!model) return;
Chris@441 762 model->setCompletion(completion, true);
Chris@441 763
Chris@441 764 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
Chris@320 765
Chris@320 766 EditableDenseThreeDimensionalModel *model =
Chris@350 767 getConformingOutput<EditableDenseThreeDimensionalModel>();
Chris@320 768 if (!model) return;
Chris@350 769 model->setCompletion(completion, true); //!!!m_context.updates);
Chris@320 770 }
Chris@320 771 }
Chris@320 772