annotate runner/FeatureExtractionManager.cpp @ 100:010fbf2a3fba

Merge from branch start-duration
author Chris Cannam
date Wed, 01 Oct 2014 14:35:01 +0100
parents 136d8496a4b8 526feaad5820
children fae326c22df5
rev   line source
Chris@0 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
Chris@0 2
Chris@0 3 /*
Chris@0 4 Sonic Annotator
Chris@0 5 A utility for batch feature extraction from audio files.
Chris@0 6 Mark Levy, Chris Sutton and Chris Cannam, Queen Mary, University of London.
Chris@0 7 Copyright 2007-2008 QMUL.
Chris@0 8
Chris@0 9 This program is free software; you can redistribute it and/or
Chris@0 10 modify it under the terms of the GNU General Public License as
Chris@0 11 published by the Free Software Foundation; either version 2 of the
Chris@0 12 License, or (at your option) any later version. See the file
Chris@0 13 COPYING included with this distribution for more information.
Chris@0 14 */
Chris@0 15
Chris@0 16 #include "FeatureExtractionManager.h"
Chris@0 17
Chris@0 18 #include <vamp-hostsdk/PluginChannelAdapter.h>
Chris@0 19 #include <vamp-hostsdk/PluginBufferingAdapter.h>
Chris@0 20 #include <vamp-hostsdk/PluginInputDomainAdapter.h>
Chris@0 21 #include <vamp-hostsdk/PluginSummarisingAdapter.h>
Chris@8 22 #include <vamp-hostsdk/PluginWrapper.h>
Chris@0 23 #include <vamp-hostsdk/PluginLoader.h>
Chris@0 24
Chris@21 25 #include "base/Exceptions.h"
Chris@21 26
Chris@0 27 #include <iostream>
Chris@0 28
Chris@0 29 using namespace std;
Chris@0 30
Chris@0 31 using Vamp::Plugin;
Chris@0 32 using Vamp::PluginBase;
Chris@0 33 using Vamp::HostExt::PluginLoader;
Chris@0 34 using Vamp::HostExt::PluginChannelAdapter;
Chris@0 35 using Vamp::HostExt::PluginBufferingAdapter;
Chris@0 36 using Vamp::HostExt::PluginInputDomainAdapter;
Chris@0 37 using Vamp::HostExt::PluginSummarisingAdapter;
Chris@8 38 using Vamp::HostExt::PluginWrapper;
Chris@0 39
Chris@0 40 #include "data/fileio/FileSource.h"
Chris@0 41 #include "data/fileio/AudioFileReader.h"
Chris@0 42 #include "data/fileio/AudioFileReaderFactory.h"
Chris@0 43 #include "data/fileio/PlaylistFileReader.h"
Chris@0 44 #include "base/TempDirectory.h"
Chris@0 45 #include "base/ProgressPrinter.h"
Chris@0 46 #include "transform/TransformFactory.h"
Chris@0 47 #include "rdf/RDFTransformFactory.h"
Chris@0 48 #include "transform/FeatureWriter.h"
Chris@0 49
Chris@0 50 #include <QTextStream>
Chris@0 51 #include <QFile>
Chris@0 52 #include <QFileInfo>
Chris@0 53
Chris@0 54 FeatureExtractionManager::FeatureExtractionManager() :
Chris@0 55 m_summariesOnly(false),
Chris@0 56 // We can read using an arbitrary fixed block size --
Chris@0 57 // PluginBufferingAdapter handles this for us. It's likely to be
Chris@0 58 // quicker to use larger sizes than smallish ones like 1024
Chris@0 59 m_blockSize(16384),
Chris@0 60 m_defaultSampleRate(0),
Chris@0 61 m_sampleRate(0),
Chris@45 62 m_channels(0)
Chris@0 63 {
Chris@0 64 }
Chris@0 65
Chris@0 66 FeatureExtractionManager::~FeatureExtractionManager()
Chris@0 67 {
Chris@0 68 for (PluginMap::iterator pi = m_plugins.begin();
Chris@0 69 pi != m_plugins.end(); ++pi) {
Chris@0 70 delete pi->first;
Chris@0 71 }
Chris@45 72 foreach (AudioFileReader *r, m_readyReaders) {
Chris@45 73 delete r;
Chris@45 74 }
Chris@0 75 }
Chris@0 76
Chris@0 77 void FeatureExtractionManager::setChannels(int channels)
Chris@0 78 {
Chris@0 79 m_channels = channels;
Chris@0 80 }
Chris@0 81
Chris@0 82 void FeatureExtractionManager::setDefaultSampleRate(int sampleRate)
Chris@0 83 {
Chris@0 84 m_defaultSampleRate = sampleRate;
Chris@0 85 }
Chris@0 86
Chris@0 87 static PluginSummarisingAdapter::SummaryType
Chris@0 88 getSummaryType(string name)
Chris@0 89 {
Chris@0 90 if (name == "min") return PluginSummarisingAdapter::Minimum;
Chris@0 91 if (name == "max") return PluginSummarisingAdapter::Maximum;
Chris@0 92 if (name == "mean") return PluginSummarisingAdapter::Mean;
Chris@0 93 if (name == "median") return PluginSummarisingAdapter::Median;
Chris@0 94 if (name == "mode") return PluginSummarisingAdapter::Mode;
Chris@0 95 if (name == "sum") return PluginSummarisingAdapter::Sum;
Chris@0 96 if (name == "variance") return PluginSummarisingAdapter::Variance;
Chris@0 97 if (name == "sd") return PluginSummarisingAdapter::StandardDeviation;
Chris@0 98 if (name == "count") return PluginSummarisingAdapter::Count;
Chris@0 99 return PluginSummarisingAdapter::UnknownSummaryType;
Chris@0 100 }
Chris@0 101
Chris@0 102 bool FeatureExtractionManager::setSummaryTypes(const set<string> &names,
Chris@0 103 bool summariesOnly,
Chris@0 104 const PluginSummarisingAdapter::SegmentBoundaries &boundaries)
Chris@0 105 {
Chris@0 106 for (SummaryNameSet::const_iterator i = names.begin();
Chris@0 107 i != names.end(); ++i) {
Chris@0 108 if (getSummaryType(*i) == PluginSummarisingAdapter::UnknownSummaryType) {
Chris@0 109 cerr << "ERROR: Unknown summary type \"" << *i << "\"" << endl;
Chris@0 110 return false;
Chris@0 111 }
Chris@0 112 }
Chris@0 113 m_summaries = names;
Chris@0 114 m_summariesOnly = summariesOnly;
Chris@0 115 m_boundaries = boundaries;
Chris@0 116 return true;
Chris@0 117 }
Chris@0 118
Chris@51 119 static PluginInputDomainAdapter::WindowType
Chris@51 120 convertWindowType(WindowType t)
Chris@51 121 {
Chris@51 122 switch (t) {
Chris@51 123 case RectangularWindow:
Chris@51 124 return PluginInputDomainAdapter::RectangularWindow;
Chris@51 125 case BartlettWindow:
Chris@51 126 return PluginInputDomainAdapter::BartlettWindow;
Chris@51 127 case HammingWindow:
Chris@51 128 return PluginInputDomainAdapter::HammingWindow;
Chris@51 129 case HanningWindow:
Chris@51 130 return PluginInputDomainAdapter::HanningWindow;
Chris@51 131 case BlackmanWindow:
Chris@51 132 return PluginInputDomainAdapter::BlackmanWindow;
Chris@51 133 case NuttallWindow:
Chris@51 134 return PluginInputDomainAdapter::NuttallWindow;
Chris@51 135 case BlackmanHarrisWindow:
Chris@51 136 return PluginInputDomainAdapter::BlackmanHarrisWindow;
Chris@51 137 default:
Chris@51 138 cerr << "ERROR: Unknown or unsupported window type \"" << t << "\", using Hann (\"" << HanningWindow << "\")" << endl;
Chris@51 139 return PluginInputDomainAdapter::HanningWindow;
Chris@51 140 }
Chris@51 141 }
Chris@51 142
Chris@0 143 bool FeatureExtractionManager::addFeatureExtractor
Chris@0 144 (Transform transform, const vector<FeatureWriter*> &writers)
Chris@0 145 {
Chris@0 146 //!!! exceptions rather than return values?
Chris@0 147
Chris@0 148 if (transform.getSampleRate() == 0) {
Chris@0 149 if (m_sampleRate == 0) {
Chris@0 150 cerr << "NOTE: Transform does not specify a sample rate, using default rate of " << m_defaultSampleRate << endl;
Chris@0 151 transform.setSampleRate(m_defaultSampleRate);
Chris@0 152 m_sampleRate = m_defaultSampleRate;
Chris@0 153 } else {
Chris@0 154 cerr << "NOTE: Transform does not specify a sample rate, using previous transform's rate of " << m_sampleRate << endl;
Chris@0 155 transform.setSampleRate(m_sampleRate);
Chris@0 156 }
Chris@0 157 }
Chris@0 158
Chris@0 159 if (m_sampleRate == 0) {
Chris@0 160 m_sampleRate = transform.getSampleRate();
Chris@0 161 }
Chris@0 162
Chris@0 163 if (transform.getSampleRate() != m_sampleRate) {
Chris@0 164 cerr << "WARNING: Transform sample rate " << transform.getSampleRate() << " does not match previously specified transform rate of " << m_sampleRate << " -- only a single rate is supported for each run" << endl;
Chris@0 165 cerr << "WARNING: Using previous rate of " << m_sampleRate << " for this transform as well" << endl;
Chris@0 166 transform.setSampleRate(m_sampleRate);
Chris@0 167 }
Chris@0 168
Chris@0 169 Plugin *plugin = 0;
Chris@0 170
Chris@0 171 // Remember what the original transform looked like, and index
Chris@0 172 // based on this -- because we may be about to fill in the zeros
Chris@0 173 // for step and block size, but we want any further copies with
Chris@0 174 // the same zeros to match this one
Chris@0 175 Transform originalTransform = transform;
Chris@0 176
Chris@0 177 if (m_transformPluginMap.find(transform) == m_transformPluginMap.end()) {
Chris@0 178
Chris@0 179 // Test whether we already have a transform that is identical
Chris@0 180 // to this, except for the output requested and/or the summary
Chris@0 181 // type -- if so, they should share plugin instances (a vital
Chris@0 182 // optimisation)
Chris@0 183
Chris@0 184 for (TransformPluginMap::iterator i = m_transformPluginMap.begin();
Chris@0 185 i != m_transformPluginMap.end(); ++i) {
Chris@0 186 Transform test = i->first;
Chris@0 187 test.setOutput(transform.getOutput());
Chris@0 188 test.setSummaryType(transform.getSummaryType());
Chris@0 189 if (transform == test) {
Chris@0 190 cerr << "NOTE: Already have transform identical to this one (for \""
Chris@0 191 << transform.getIdentifier().toStdString()
Chris@0 192 << "\") in every detail except output identifier and/or "
Chris@0 193 << "summary type; sharing its plugin instance" << endl;
Chris@0 194 plugin = i->second;
Chris@0 195 if (transform.getSummaryType() != Transform::NoSummary &&
Chris@0 196 !dynamic_cast<PluginSummarisingAdapter *>(plugin)) {
Chris@0 197 plugin = new PluginSummarisingAdapter(plugin);
Chris@0 198 i->second = plugin;
Chris@0 199 }
Chris@0 200 break;
Chris@0 201 }
Chris@0 202 }
Chris@0 203
Chris@0 204 if (!plugin) {
Chris@0 205
Chris@0 206 TransformFactory *tf = TransformFactory::getInstance();
Chris@0 207
Chris@0 208 PluginBase *pb = tf->instantiatePluginFor(transform);
Chris@0 209 plugin = tf->downcastVampPlugin(pb);
Chris@0 210 if (!plugin) {
Chris@0 211 //!!! todo: handle non-Vamp plugins too, or make the main --list
Chris@0 212 // option print out only Vamp transforms
Chris@0 213 cerr << "ERROR: Failed to load plugin for transform \""
Chris@0 214 << transform.getIdentifier().toStdString() << "\"" << endl;
Chris@0 215 delete pb;
Chris@0 216 return false;
Chris@0 217 }
Chris@0 218
Chris@0 219 // We will provide the plugin with arbitrary step and
Chris@0 220 // block sizes (so that we can use the same read/write
Chris@0 221 // block size for all transforms), and to that end we use
Chris@0 222 // a PluginBufferingAdapter. However, we need to know the
Chris@0 223 // underlying step size so that we can provide the right
Chris@0 224 // context for dense outputs. (Although, don't forget
Chris@0 225 // that the PluginBufferingAdapter rewrites
Chris@0 226 // OneSamplePerStep outputs so as to use FixedSampleRate
Chris@0 227 // -- so it supplies the sample rate in the output
Chris@0 228 // feature. I'm not sure whether we can easily use that.)
Chris@0 229
Chris@0 230 size_t pluginStepSize = plugin->getPreferredStepSize();
Chris@0 231 size_t pluginBlockSize = plugin->getPreferredBlockSize();
Chris@0 232
Chris@25 233 PluginInputDomainAdapter *pida = 0;
Chris@25 234
Chris@0 235 // adapt the plugin for buffering, channels, etc.
Chris@0 236 if (plugin->getInputDomain() == Plugin::FrequencyDomain) {
Chris@51 237
Chris@25 238 pida = new PluginInputDomainAdapter(plugin);
Chris@26 239 pida->setProcessTimestampMethod(PluginInputDomainAdapter::ShiftData);
Chris@51 240
Chris@51 241 PluginInputDomainAdapter::WindowType wtype =
Chris@51 242 convertWindowType(transform.getWindowType());
Chris@51 243 pida->setWindowType(wtype);
Chris@25 244 plugin = pida;
Chris@0 245 }
Chris@0 246
Chris@0 247 PluginBufferingAdapter *pba = new PluginBufferingAdapter(plugin);
Chris@0 248 plugin = pba;
Chris@0 249
Chris@0 250 if (transform.getStepSize() != 0) {
Chris@0 251 pba->setPluginStepSize(transform.getStepSize());
Chris@0 252 } else {
Chris@0 253 transform.setStepSize(pluginStepSize);
Chris@0 254 }
Chris@0 255
Chris@0 256 if (transform.getBlockSize() != 0) {
Chris@0 257 pba->setPluginBlockSize(transform.getBlockSize());
Chris@0 258 } else {
Chris@0 259 transform.setBlockSize(pluginBlockSize);
Chris@0 260 }
Chris@0 261
Chris@0 262 plugin = new PluginChannelAdapter(plugin);
Chris@0 263
Chris@0 264 if (!m_summaries.empty() ||
Chris@0 265 transform.getSummaryType() != Transform::NoSummary) {
Chris@0 266 PluginSummarisingAdapter *adapter =
Chris@0 267 new PluginSummarisingAdapter(plugin);
Chris@0 268 adapter->setSummarySegmentBoundaries(m_boundaries);
Chris@0 269 plugin = adapter;
Chris@0 270 }
Chris@0 271
Chris@0 272 if (!plugin->initialise(m_channels, m_blockSize, m_blockSize)) {
Chris@0 273 cerr << "ERROR: Plugin initialise (channels = " << m_channels << ", stepSize = " << m_blockSize << ", blockSize = " << m_blockSize << ") failed." << endl;
Chris@0 274 delete plugin;
Chris@0 275 return false;
Chris@0 276 }
Chris@0 277
Chris@0 278 // cerr << "Initialised plugin" << endl;
Chris@0 279
Chris@0 280 size_t actualStepSize = 0;
Chris@0 281 size_t actualBlockSize = 0;
Chris@0 282 pba->getActualStepAndBlockSizes(actualStepSize, actualBlockSize);
Chris@0 283 transform.setStepSize(actualStepSize);
Chris@0 284 transform.setBlockSize(actualBlockSize);
Chris@0 285
Chris@0 286 Plugin::OutputList outputs = plugin->getOutputDescriptors();
Chris@0 287 for (int i = 0; i < (int)outputs.size(); ++i) {
Chris@0 288
Chris@0 289 // cerr << "Newly initialised plugin output " << i << " has bin count " << outputs[i].binCount << endl;
Chris@0 290
Chris@0 291 m_pluginOutputs[plugin][outputs[i].identifier] = outputs[i];
Chris@0 292 m_pluginOutputIndices[outputs[i].identifier] = i;
Chris@0 293 }
Chris@0 294
Chris@10 295 cerr << "NOTE: Loaded and initialised plugin for transform \""
Chris@25 296 << transform.getIdentifier().toStdString()
Chris@25 297 << "\" with plugin step size " << actualStepSize
Chris@25 298 << " and block size " << actualBlockSize
Chris@25 299 << " (adapter step and block size " << m_blockSize << ")"
Chris@25 300 << endl;
Chris@25 301
Chris@25 302 if (pida) {
Chris@25 303 cerr << "NOTE: PluginInputDomainAdapter timestamp adjustment is "
Chris@25 304
Chris@25 305 << pida->getTimestampAdjustment() << endl;
Chris@25 306 }
Chris@8 307
Chris@8 308 } else {
Chris@8 309
Chris@8 310 if (transform.getStepSize() == 0 || transform.getBlockSize() == 0) {
Chris@8 311
Chris@8 312 PluginWrapper *pw = dynamic_cast<PluginWrapper *>(plugin);
Chris@8 313 if (pw) {
Chris@8 314 PluginBufferingAdapter *pba =
Chris@8 315 pw->getWrapper<PluginBufferingAdapter>();
Chris@8 316 if (pba) {
Chris@8 317 size_t actualStepSize = 0;
Chris@8 318 size_t actualBlockSize = 0;
Chris@8 319 pba->getActualStepAndBlockSizes(actualStepSize,
Chris@8 320 actualBlockSize);
Chris@8 321 if (transform.getStepSize() == 0) {
Chris@8 322 transform.setStepSize(actualStepSize);
Chris@8 323 }
Chris@8 324 if (transform.getBlockSize() == 0) {
Chris@8 325 transform.setBlockSize(actualBlockSize);
Chris@8 326 }
Chris@8 327 }
Chris@8 328 }
Chris@8 329 }
Chris@0 330 }
Chris@0 331
Chris@0 332 if (transform.getOutput() == "") {
Chris@0 333 transform.setOutput
Chris@0 334 (plugin->getOutputDescriptors()[0].identifier.c_str());
Chris@0 335 }
Chris@0 336
Chris@0 337 m_transformPluginMap[transform] = plugin;
Chris@0 338
Chris@0 339 if (!(originalTransform == transform)) {
Chris@0 340 m_transformPluginMap[originalTransform] = plugin;
Chris@0 341 }
Chris@0 342
Chris@0 343 } else {
Chris@0 344
Chris@0 345 plugin = m_transformPluginMap[transform];
Chris@0 346 }
Chris@0 347
Chris@0 348 m_plugins[plugin][transform] = writers;
Chris@0 349
Chris@0 350 return true;
Chris@0 351 }
Chris@0 352
Chris@0 353 bool FeatureExtractionManager::addDefaultFeatureExtractor
Chris@0 354 (TransformId transformId, const vector<FeatureWriter*> &writers)
Chris@0 355 {
Chris@0 356 TransformFactory *tf = TransformFactory::getInstance();
Chris@0 357
Chris@0 358 if (m_sampleRate == 0) {
Chris@0 359 if (m_defaultSampleRate == 0) {
Chris@0 360 cerr << "ERROR: Default transform requested, but no default sample rate available" << endl;
Chris@0 361 return false;
Chris@0 362 } else {
Chris@0 363 cerr << "NOTE: Using default sample rate of " << m_defaultSampleRate << " for default transform" << endl;
Chris@0 364 m_sampleRate = m_defaultSampleRate;
Chris@0 365 }
Chris@0 366 }
Chris@0 367
Chris@0 368 Transform transform = tf->getDefaultTransformFor(transformId, m_sampleRate);
Chris@0 369
Chris@0 370 return addFeatureExtractor(transform, writers);
Chris@0 371 }
Chris@0 372
Chris@0 373 bool FeatureExtractionManager::addFeatureExtractorFromFile
Chris@0 374 (QString transformXmlFile, const vector<FeatureWriter*> &writers)
Chris@0 375 {
Chris@99 376 bool tryRdf = true;
Chris@99 377
Chris@99 378 if (transformXmlFile.endsWith(".xml") || transformXmlFile.endsWith(".XML")) {
Chris@99 379 // We don't support RDF-XML (and nor does the underlying
Chris@99 380 // parser library) so skip the RDF parse if the filename
Chris@99 381 // suggests XML, to avoid puking out a load of errors from
Chris@99 382 // feeding XML to a Turtle parser
Chris@99 383 tryRdf = false;
Chris@0 384 }
Chris@99 385
Chris@99 386 if (tryRdf) {
Chris@99 387 RDFTransformFactory factory
Chris@99 388 (QUrl::fromLocalFile(QFileInfo(transformXmlFile).absoluteFilePath())
Chris@99 389 .toString());
Chris@99 390 ProgressPrinter printer("Parsing transforms RDF file");
Chris@99 391 std::vector<Transform> transforms = factory.getTransforms(&printer);
Chris@99 392 if (!factory.isOK()) {
Chris@99 393 cerr << "WARNING: FeatureExtractionManager::addFeatureExtractorFromFile: Failed to parse transforms file: " << factory.getErrorString().toStdString() << endl;
Chris@99 394 if (factory.isRDF()) {
Chris@99 395 return false; // no point trying it as XML
Chris@0 396 }
Chris@0 397 }
Chris@99 398 if (!transforms.empty()) {
Chris@99 399 bool success = true;
Chris@99 400 for (int i = 0; i < (int)transforms.size(); ++i) {
Chris@99 401 if (!addFeatureExtractor(transforms[i], writers)) {
Chris@99 402 success = false;
Chris@99 403 }
Chris@99 404 }
Chris@99 405 return success;
Chris@99 406 }
Chris@0 407 }
Chris@0 408
Chris@0 409 QFile file(transformXmlFile);
Chris@0 410 if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@0 411 cerr << "ERROR: Failed to open transform XML file \""
Chris@0 412 << transformXmlFile.toStdString() << "\" for reading" << endl;
Chris@0 413 return false;
Chris@0 414 }
Chris@0 415
Chris@0 416 QTextStream *qts = new QTextStream(&file);
Chris@0 417 QString qs = qts->readAll();
Chris@0 418 delete qts;
Chris@0 419 file.close();
Chris@0 420
Chris@0 421 Transform transform(qs);
Chris@0 422
Chris@0 423 return addFeatureExtractor(transform, writers);
Chris@0 424 }
Chris@0 425
Chris@45 426 void FeatureExtractionManager::addSource(QString audioSource)
Chris@0 427 {
Chris@0 428 if (QFileInfo(audioSource).suffix().toLower() == "m3u") {
Chris@45 429 ProgressPrinter retrievalProgress("Opening playlist file...");
Chris@45 430 FileSource source(audioSource, &retrievalProgress);
Chris@45 431 if (!source.isAvailable()) {
Chris@45 432 cerr << "ERROR: File or URL \"" << audioSource.toStdString()
Chris@45 433 << "\" could not be located" << endl;
Chris@45 434 throw FileNotFound(audioSource);
Chris@45 435 }
Chris@45 436 source.waitForData();
Chris@0 437 PlaylistFileReader reader(source);
Chris@0 438 if (reader.isOK()) {
Chris@0 439 vector<QString> files = reader.load();
Chris@0 440 for (int i = 0; i < (int)files.size(); ++i) {
Chris@45 441 addSource(files[i]);
Chris@0 442 }
Chris@0 443 return;
Chris@0 444 } else {
Chris@0 445 cerr << "ERROR: Playlist \"" << audioSource.toStdString()
Chris@0 446 << "\" could not be opened" << endl;
Chris@21 447 throw FileNotFound(audioSource);
Chris@0 448 }
Chris@0 449 }
Chris@0 450
Chris@45 451 std::cerr << "Have audio source: \"" << audioSource.toStdString() << "\"" << std::endl;
Chris@45 452
Chris@45 453 // We don't actually do anything with it here, unless it's the
Chris@45 454 // first audio source and we need it to establish default channel
Chris@45 455 // count and sample rate
Chris@45 456
Chris@45 457 if (m_channels == 0 || m_defaultSampleRate == 0) {
Chris@45 458
Chris@45 459 ProgressPrinter retrievalProgress("Determining default rate and channel count from first input file...");
Chris@45 460
Chris@45 461 FileSource source(audioSource, &retrievalProgress);
Chris@45 462 if (!source.isAvailable()) {
Chris@45 463 cerr << "ERROR: File or URL \"" << audioSource.toStdString()
Chris@45 464 << "\" could not be located" << endl;
Chris@45 465 throw FileNotFound(audioSource);
Chris@45 466 }
Chris@45 467
Chris@45 468 source.waitForData();
Chris@45 469
Chris@45 470 // Open to determine validity, channel count, sample rate only
Chris@45 471 // (then close, and open again later with actual desired rate &c)
Chris@45 472
Chris@45 473 AudioFileReader *reader =
Chris@95 474 AudioFileReaderFactory::createReader(source, 0, false,
Chris@95 475 &retrievalProgress);
Chris@45 476
Chris@45 477 if (!reader) {
Chris@45 478 throw FailedToOpenFile(audioSource);
Chris@45 479 }
Chris@45 480
Chris@45 481 retrievalProgress.done();
Chris@45 482
Chris@45 483 cerr << "File or URL \"" << audioSource.toStdString() << "\" opened successfully" << endl;
Chris@45 484
Chris@45 485 if (m_channels == 0) {
Chris@45 486 m_channels = reader->getChannelCount();
Chris@45 487 cerr << "Taking default channel count of "
Chris@45 488 << reader->getChannelCount() << " from file" << endl;
Chris@45 489 }
Chris@45 490
Chris@45 491 if (m_defaultSampleRate == 0) {
Chris@45 492 m_defaultSampleRate = reader->getNativeRate();
Chris@45 493 cerr << "Taking default sample rate of "
Chris@45 494 << reader->getNativeRate() << "Hz from file" << endl;
Chris@45 495 cerr << "(Note: Default may be overridden by transforms)" << endl;
Chris@45 496 }
Chris@45 497
Chris@45 498 m_readyReaders[audioSource] = reader;
Chris@45 499 }
Chris@45 500 }
Chris@45 501
Chris@47 502 void FeatureExtractionManager::extractFeatures(QString audioSource, bool force)
Chris@45 503 {
Chris@45 504 if (m_plugins.empty()) return;
Chris@45 505
Chris@47 506 if (QFileInfo(audioSource).suffix().toLower() == "m3u") {
Chris@47 507 FileSource source(audioSource);
Chris@47 508 PlaylistFileReader reader(source);
Chris@47 509 if (reader.isOK()) {
Chris@47 510 vector<QString> files = reader.load();
Chris@47 511 for (int i = 0; i < (int)files.size(); ++i) {
Chris@47 512 try {
Chris@47 513 extractFeatures(files[i], force);
Chris@47 514 } catch (const std::exception &e) {
Chris@47 515 if (!force) throw;
Chris@47 516 cerr << "ERROR: Feature extraction failed for playlist entry \""
Chris@47 517 << files[i].toStdString()
Chris@47 518 << "\": " << e.what() << endl;
Chris@47 519 // print a note only if we have more files to process
Chris@95 520 if (++i != (int)files.size()) {
Chris@47 521 cerr << "NOTE: \"--force\" option was provided, continuing (more errors may occur)" << endl;
Chris@47 522 }
Chris@47 523 }
Chris@47 524 }
Chris@47 525 return;
Chris@47 526 } else {
Chris@47 527 cerr << "ERROR: Playlist \"" << audioSource.toStdString()
Chris@47 528 << "\" could not be opened" << endl;
Chris@47 529 throw FileNotFound(audioSource);
Chris@47 530 }
Chris@47 531 }
Chris@47 532
Chris@45 533 testOutputFiles(audioSource);
Chris@45 534
Chris@0 535 if (m_sampleRate == 0) {
Chris@45 536 throw FileOperationFailed
Chris@45 537 (audioSource, "internal error: have sources and plugins, but no sample rate");
Chris@45 538 }
Chris@45 539 if (m_channels == 0) {
Chris@45 540 throw FileOperationFailed
Chris@45 541 (audioSource, "internal error: have sources and plugins, but no channel count");
Chris@0 542 }
Chris@0 543
Chris@45 544 AudioFileReader *reader = 0;
Chris@45 545
Chris@45 546 if (m_readyReaders.contains(audioSource)) {
Chris@45 547 reader = m_readyReaders[audioSource];
Chris@45 548 m_readyReaders.remove(audioSource);
Chris@45 549 if (reader->getChannelCount() != m_channels ||
Chris@45 550 reader->getSampleRate() != m_sampleRate) {
Chris@45 551 // can't use this; open it again
Chris@45 552 delete reader;
Chris@45 553 reader = 0;
Chris@45 554 }
Chris@45 555 }
Chris@45 556 if (!reader) {
Chris@45 557 ProgressPrinter retrievalProgress("Retrieving audio data...");
Chris@45 558 FileSource source(audioSource, &retrievalProgress);
Chris@45 559 source.waitForData();
Chris@45 560 reader = AudioFileReaderFactory::createReader
Chris@95 561 (source, m_sampleRate, false, &retrievalProgress);
Chris@45 562 retrievalProgress.done();
Chris@45 563 }
Chris@45 564
Chris@0 565 if (!reader) {
Chris@21 566 throw FailedToOpenFile(audioSource);
Chris@0 567 }
Chris@0 568
Chris@45 569 cerr << "Audio file \"" << audioSource.toStdString() << "\": "
Chris@45 570 << reader->getChannelCount() << "ch at "
Chris@45 571 << reader->getNativeRate() << "Hz" << endl;
Chris@45 572 if (reader->getChannelCount() != m_channels ||
Chris@45 573 reader->getNativeRate() != m_sampleRate) {
Chris@45 574 cerr << "NOTE: File will be mixed or resampled for processing: "
Chris@45 575 << m_channels << "ch at "
Chris@45 576 << m_sampleRate << "Hz" << endl;
Chris@45 577 }
Chris@11 578
Chris@0 579 // allocate audio buffers
Chris@0 580 float **data = new float *[m_channels];
Chris@0 581 for (int c = 0; c < m_channels; ++c) {
Chris@0 582 data[c] = new float[m_blockSize];
Chris@0 583 }
Chris@31 584
Chris@31 585 struct LifespanMgr { // unintrusive hack introduced to ensure
Chris@31 586 // destruction on exceptions
Chris@31 587 AudioFileReader *m_r;
Chris@31 588 int m_c;
Chris@31 589 float **m_d;
Chris@31 590 LifespanMgr(AudioFileReader *r, int c, float **d) :
Chris@31 591 m_r(r), m_c(c), m_d(d) { }
Chris@31 592 ~LifespanMgr() { destroy(); }
Chris@31 593 void destroy() {
Chris@31 594 if (!m_r) return;
Chris@31 595 delete m_r;
Chris@31 596 for (int i = 0; i < m_c; ++i) delete[] m_d[i];
Chris@31 597 delete[] m_d;
Chris@31 598 m_r = 0;
Chris@31 599 }
Chris@31 600 };
Chris@31 601 LifespanMgr lifemgr(reader, m_channels, data);
Chris@0 602
Chris@0 603 size_t frameCount = reader->getFrameCount();
Chris@0 604
Chris@0 605 // cerr << "file has " << frameCount << " frames" << endl;
Chris@0 606
Chris@99 607 int earliestStartFrame = 0;
Chris@99 608 int latestEndFrame = frameCount;
Chris@99 609 bool haveExtents = false;
Chris@99 610
Chris@0 611 for (PluginMap::iterator pi = m_plugins.begin();
Chris@0 612 pi != m_plugins.end(); ++pi) {
Chris@0 613
Chris@0 614 Plugin *plugin = pi->first;
Chris@0 615
Chris@0 616 // std::cerr << "Calling reset on " << plugin << std::endl;
Chris@0 617 plugin->reset();
Chris@0 618
Chris@0 619 for (TransformWriterMap::iterator ti = pi->second.begin();
Chris@0 620 ti != pi->second.end(); ++ti) {
Chris@0 621
Chris@0 622 const Transform &transform = ti->first;
Chris@0 623
Chris@99 624 int startFrame = RealTime::realTime2Frame
Chris@99 625 (transform.getStartTime(), m_sampleRate);
Chris@99 626 int duration = RealTime::realTime2Frame
Chris@99 627 (transform.getDuration(), m_sampleRate);
Chris@99 628 if (duration == 0) {
Chris@99 629 duration = frameCount - startFrame;
Chris@99 630 }
Chris@99 631
Chris@99 632 if (!haveExtents || startFrame < earliestStartFrame) {
Chris@99 633 earliestStartFrame = startFrame;
Chris@99 634 }
Chris@99 635 if (!haveExtents || startFrame + duration > latestEndFrame) {
Chris@99 636 latestEndFrame = startFrame + duration;
Chris@99 637 }
Chris@99 638
Chris@99 639 haveExtents = true;
Chris@0 640
Chris@0 641 string outputId = transform.getOutput().toStdString();
Chris@0 642 if (m_pluginOutputs[plugin].find(outputId) ==
Chris@0 643 m_pluginOutputs[plugin].end()) {
Chris@0 644 //!!! throw?
Chris@0 645 cerr << "WARNING: Nonexistent plugin output \"" << outputId << "\" requested for transform \""
Chris@0 646 << transform.getIdentifier().toStdString() << "\", ignoring this transform"
Chris@0 647 << endl;
Chris@0 648 /*
Chris@0 649 cerr << "Known outputs for all plugins are as follows:" << endl;
Chris@0 650 for (PluginOutputMap::const_iterator k = m_pluginOutputs.begin();
Chris@0 651 k != m_pluginOutputs.end(); ++k) {
Chris@0 652 cerr << "Plugin " << k->first << ": ";
Chris@0 653 if (k->second.empty()) {
Chris@0 654 cerr << "(none)";
Chris@0 655 }
Chris@0 656 for (OutputMap::const_iterator i = k->second.begin();
Chris@0 657 i != k->second.end(); ++i) {
Chris@0 658 cerr << "\"" << i->first << "\" ";
Chris@0 659 }
Chris@0 660 cerr << endl;
Chris@0 661 }
Chris@0 662 */
Chris@0 663 }
Chris@0 664 }
Chris@0 665 }
Chris@0 666
Chris@99 667 int startFrame = earliestStartFrame;
Chris@99 668 int endFrame = latestEndFrame;
Chris@0 669
Chris@0 670 for (PluginMap::iterator pi = m_plugins.begin();
Chris@0 671 pi != m_plugins.end(); ++pi) {
Chris@0 672
Chris@0 673 for (TransformWriterMap::const_iterator ti = pi->second.begin();
Chris@0 674 ti != pi->second.end(); ++ti) {
Chris@0 675
Chris@0 676 const vector<FeatureWriter *> &writers = ti->second;
Chris@0 677
Chris@0 678 for (int j = 0; j < (int)writers.size(); ++j) {
Chris@0 679 FeatureWriter::TrackMetadata m;
Chris@0 680 m.title = reader->getTitle();
Chris@0 681 m.maker = reader->getMaker();
Chris@19 682 if (m.title != "" && m.maker != "") {
Chris@19 683 writers[j]->setTrackMetadata(audioSource, m);
Chris@19 684 }
Chris@0 685 }
Chris@0 686 }
Chris@0 687 }
Chris@0 688
Chris@0 689 ProgressPrinter extractionProgress("Extracting and writing features...");
Chris@0 690 int progress = 0;
Chris@0 691
Chris@99 692 for (int i = startFrame; i < endFrame; i += m_blockSize) {
Chris@0 693
Chris@0 694 //!!! inefficient, although much of the inefficiency may be
Chris@99 695 // susceptible to compiler optimisation
Chris@0 696
Chris@0 697 SampleBlock frames;
Chris@0 698 reader->getInterleavedFrames(i, m_blockSize, frames);
Chris@0 699
Chris@0 700 // We have to do our own channel handling here; we can't just
Chris@0 701 // leave it to the plugin adapter because the same plugin
Chris@0 702 // adapter may have to serve for input files with various
Chris@0 703 // numbers of channels (so the adapter is simply configured
Chris@34 704 // with a fixed channel count).
Chris@0 705
Chris@0 706 int rc = reader->getChannelCount();
Chris@0 707
Chris@34 708 // m_channels is the number of channels we need for the plugin
Chris@34 709
Chris@34 710 int index;
Chris@34 711 int fc = (int)frames.size();
Chris@46 712
Chris@34 713 if (m_channels == 1) { // only case in which we can sensibly mix down
Chris@34 714 for (int j = 0; j < m_blockSize; ++j) {
Chris@34 715 data[0][j] = 0.f;
Chris@34 716 }
Chris@34 717 for (int c = 0; c < rc; ++c) {
Chris@34 718 for (int j = 0; j < m_blockSize; ++j) {
Chris@0 719 index = j * rc + c;
Chris@34 720 if (index < fc) data[0][j] += frames[index];
Chris@0 721 }
Chris@0 722 }
Chris@34 723 for (int j = 0; j < m_blockSize; ++j) {
Chris@34 724 data[0][j] /= rc;
Chris@34 725 }
Chris@34 726 } else {
Chris@34 727 for (int c = 0; c < m_channels; ++c) {
Chris@34 728 for (int j = 0; j < m_blockSize; ++j) {
Chris@34 729 data[c][j] = 0.f;
Chris@34 730 }
Chris@34 731 if (c < rc) {
Chris@34 732 for (int j = 0; j < m_blockSize; ++j) {
Chris@34 733 index = j * rc + c;
Chris@34 734 if (index < fc) data[c][j] += frames[index];
Chris@34 735 }
Chris@34 736 }
Chris@34 737 }
Chris@34 738 }
Chris@0 739
Chris@0 740 Vamp::RealTime timestamp = Vamp::RealTime::frame2RealTime
Chris@0 741 (i, m_sampleRate);
Chris@0 742
Chris@0 743 for (PluginMap::iterator pi = m_plugins.begin();
Chris@0 744 pi != m_plugins.end(); ++pi) {
Chris@0 745
Chris@0 746 Plugin *plugin = pi->first;
Chris@0 747 Plugin::FeatureSet featureSet = plugin->process(data, timestamp);
Chris@0 748
Chris@0 749 if (!m_summariesOnly) {
Chris@0 750 writeFeatures(audioSource, plugin, featureSet);
Chris@0 751 }
Chris@0 752 }
Chris@0 753
Chris@0 754 int pp = progress;
Chris@6 755 progress = int(((i - startFrame) * 100.0) / (endFrame - startFrame) + 0.1);
Chris@0 756 if (progress > pp) extractionProgress.setProgress(progress);
Chris@0 757 }
Chris@10 758
Chris@22 759 // std::cerr << "FeatureExtractionManager: deleting audio file reader" << std::endl;
Chris@12 760
Chris@31 761 lifemgr.destroy(); // deletes reader, data
Chris@57 762
Chris@57 763 // In order to ensure our results are written to the output in a
Chris@57 764 // fixed order (and not one that depends on the pointer value of
Chris@57 765 // each plugin on the heap in any given run of the program) we
Chris@57 766 // take the plugins' entries from the plugin map and sort them
Chris@57 767 // into a new, temporary map that is indexed by the first
Chris@57 768 // transform for each plugin. We then iterate over than instead of
Chris@57 769 // over m_plugins in order to get the right ordering.
Chris@57 770
Chris@57 771 // This is not the most elegant way to do this -- it would be more
Chris@57 772 // elegant to impose an ordering directly on the plugins that are
Chris@57 773 // used as keys to m_plugins. But the plugin type comes from the
Chris@57 774 // Vamp SDK, so this change is more localised.
Chris@57 775
Chris@57 776 // Thanks to Matthias for this.
Chris@57 777
Chris@58 778 typedef map<Transform, PluginMap::value_type> OrderedPluginMap;
Chris@58 779 OrderedPluginMap orderedPlugins;
Chris@57 780
Chris@0 781 for (PluginMap::iterator pi = m_plugins.begin();
Chris@0 782 pi != m_plugins.end(); ++pi) {
Chris@57 783 Transform firstForPlugin = (pi->second).begin()->first;
Chris@58 784 orderedPlugins.insert(OrderedPluginMap::value_type(firstForPlugin, *pi));
Chris@57 785 }
Chris@0 786
Chris@58 787 for (OrderedPluginMap::iterator superPi = orderedPlugins.begin();
Chris@57 788 superPi != orderedPlugins.end(); ++superPi) {
Chris@57 789
Chris@57 790 // The value we extract from this map is just the same as the
Chris@57 791 // value_type we get from iterating over our PluginMap
Chris@57 792 // directly -- but we happen to get them in the right order
Chris@57 793 // now because the map iterator is ordered by the Transform
Chris@57 794 // key type ordering
Chris@58 795 PluginMap::value_type pi = superPi->second;
Chris@57 796
Chris@57 797 Plugin *plugin = pi.first;
Chris@0 798 Plugin::FeatureSet featureSet = plugin->getRemainingFeatures();
Chris@0 799
Chris@0 800 if (!m_summariesOnly) {
Chris@0 801 writeFeatures(audioSource, plugin, featureSet);
Chris@0 802 }
Chris@0 803
Chris@0 804 if (!m_summaries.empty()) {
Chris@0 805 PluginSummarisingAdapter *adapter =
Chris@0 806 dynamic_cast<PluginSummarisingAdapter *>(plugin);
Chris@0 807 if (!adapter) {
Chris@0 808 cerr << "WARNING: Summaries requested, but plugin is not a summarising adapter" << endl;
Chris@0 809 } else {
Chris@0 810 for (SummaryNameSet::const_iterator sni = m_summaries.begin();
Chris@0 811 sni != m_summaries.end(); ++sni) {
Chris@0 812 featureSet.clear();
Chris@0 813 //!!! problem here -- we are requesting summaries
Chris@0 814 //!!! for all outputs, but they in principle have
Chris@0 815 //!!! different averaging requirements depending
Chris@0 816 //!!! on whether their features have duration or
Chris@0 817 //!!! not
Chris@0 818 featureSet = adapter->getSummaryForAllOutputs
Chris@0 819 (getSummaryType(*sni),
Chris@0 820 PluginSummarisingAdapter::ContinuousTimeAverage);
Chris@0 821 writeFeatures(audioSource, plugin, featureSet,//!!! *sni);
Chris@0 822 Transform::stringToSummaryType(sni->c_str()));
Chris@0 823 }
Chris@0 824 }
Chris@0 825 }
Chris@0 826
Chris@0 827 writeSummaries(audioSource, plugin);
Chris@0 828 }
Chris@0 829
Chris@3 830 extractionProgress.done();
Chris@3 831
Chris@0 832 finish();
Chris@0 833
Chris@0 834 TempDirectory::getInstance()->cleanup();
Chris@0 835 }
Chris@0 836
Chris@0 837 void
Chris@0 838 FeatureExtractionManager::writeSummaries(QString audioSource, Plugin *plugin)
Chris@0 839 {
Chris@0 840 // caller should have ensured plugin is in m_plugins
Chris@0 841 PluginMap::iterator pi = m_plugins.find(plugin);
Chris@0 842
Chris@0 843 for (TransformWriterMap::const_iterator ti = pi->second.begin();
Chris@0 844 ti != pi->second.end(); ++ti) {
Chris@0 845
Chris@0 846 const Transform &transform = ti->first;
Chris@0 847 const vector<FeatureWriter *> &writers = ti->second;
Chris@0 848
Chris@0 849 Transform::SummaryType summaryType = transform.getSummaryType();
Chris@0 850 PluginSummarisingAdapter::SummaryType pType =
Chris@0 851 (PluginSummarisingAdapter::SummaryType)summaryType;
Chris@0 852
Chris@0 853 if (transform.getSummaryType() == Transform::NoSummary) {
Chris@0 854 continue;
Chris@0 855 }
Chris@0 856
Chris@0 857 PluginSummarisingAdapter *adapter =
Chris@0 858 dynamic_cast<PluginSummarisingAdapter *>(plugin);
Chris@0 859 if (!adapter) {
Chris@0 860 cerr << "FeatureExtractionManager::writeSummaries: INTERNAL ERROR: Summary requested for transform, but plugin is not a summarising adapter" << endl;
Chris@0 861 continue;
Chris@0 862 }
Chris@0 863
Chris@0 864 Plugin::FeatureSet featureSet = adapter->getSummaryForAllOutputs
Chris@0 865 (pType, PluginSummarisingAdapter::ContinuousTimeAverage);
Chris@0 866
Chris@0 867 // cout << "summary type " << int(pType) << " for transform:" << endl << transform.toXmlString().toStdString()<< endl << "... feature set with " << featureSet.size() << " elts" << endl;
Chris@0 868
Chris@0 869 writeFeatures(audioSource, plugin, featureSet, summaryType);
Chris@0 870 }
Chris@0 871 }
Chris@0 872
Chris@0 873 void FeatureExtractionManager::writeFeatures(QString audioSource,
Chris@0 874 Plugin *plugin,
Chris@0 875 const Plugin::FeatureSet &features,
Chris@0 876 Transform::SummaryType summaryType)
Chris@0 877 {
Chris@0 878 // caller should have ensured plugin is in m_plugins
Chris@0 879 PluginMap::iterator pi = m_plugins.find(plugin);
Chris@0 880
Chris@0 881 for (TransformWriterMap::const_iterator ti = pi->second.begin();
Chris@0 882 ti != pi->second.end(); ++ti) {
Chris@0 883
Chris@0 884 const Transform &transform = ti->first;
Chris@0 885 const vector<FeatureWriter *> &writers = ti->second;
Chris@0 886
Chris@0 887 if (transform.getSummaryType() != Transform::NoSummary &&
Chris@0 888 m_summaries.empty() &&
Chris@0 889 summaryType == Transform::NoSummary) {
Chris@0 890 continue;
Chris@0 891 }
Chris@0 892
Chris@0 893 if (transform.getSummaryType() != Transform::NoSummary &&
Chris@0 894 summaryType != Transform::NoSummary &&
Chris@0 895 transform.getSummaryType() != summaryType) {
Chris@0 896 continue;
Chris@0 897 }
Chris@0 898
Chris@0 899 string outputId = transform.getOutput().toStdString();
Chris@0 900
Chris@0 901 if (m_pluginOutputs[plugin].find(outputId) ==
Chris@0 902 m_pluginOutputs[plugin].end()) {
Chris@0 903 continue;
Chris@0 904 }
Chris@0 905
Chris@0 906 const Plugin::OutputDescriptor &desc =
Chris@0 907 m_pluginOutputs[plugin][outputId];
Chris@0 908
Chris@0 909 int outputIndex = m_pluginOutputIndices[outputId];
Chris@0 910 Plugin::FeatureSet::const_iterator fsi = features.find(outputIndex);
Chris@0 911 if (fsi == features.end()) continue;
Chris@0 912
Chris@0 913 for (int j = 0; j < (int)writers.size(); ++j) {
Chris@0 914 writers[j]->write
Chris@0 915 (audioSource, transform, desc, fsi->second,
Chris@0 916 Transform::summaryTypeToString(summaryType).toStdString());
Chris@0 917 }
Chris@0 918 }
Chris@0 919 }
Chris@0 920
Chris@31 921 void FeatureExtractionManager::testOutputFiles(QString audioSource)
Chris@31 922 {
Chris@31 923 for (PluginMap::iterator pi = m_plugins.begin();
Chris@31 924 pi != m_plugins.end(); ++pi) {
Chris@31 925
Chris@31 926 for (TransformWriterMap::iterator ti = pi->second.begin();
Chris@31 927 ti != pi->second.end(); ++ti) {
Chris@31 928
Chris@31 929 vector<FeatureWriter *> &writers = ti->second;
Chris@31 930
Chris@31 931 for (int i = 0; i < (int)writers.size(); ++i) {
Chris@31 932 writers[i]->testOutputFile(audioSource, ti->first.getIdentifier());
Chris@31 933 }
Chris@31 934 }
Chris@31 935 }
Chris@31 936 }
Chris@31 937
Chris@0 938 void FeatureExtractionManager::finish()
Chris@0 939 {
Chris@0 940 for (PluginMap::iterator pi = m_plugins.begin();
Chris@0 941 pi != m_plugins.end(); ++pi) {
Chris@0 942
Chris@0 943 for (TransformWriterMap::iterator ti = pi->second.begin();
Chris@0 944 ti != pi->second.end(); ++ti) {
Chris@0 945
Chris@0 946 vector<FeatureWriter *> &writers = ti->second;
Chris@0 947
Chris@0 948 for (int i = 0; i < (int)writers.size(); ++i) {
Chris@0 949 writers[i]->flush();
Chris@0 950 writers[i]->finish();
Chris@0 951 }
Chris@0 952 }
Chris@0 953 }
Chris@0 954 }
Chris@0 955
Chris@0 956 void FeatureExtractionManager::print(Transform transform) const
Chris@0 957 {
Chris@0 958 QString qs;
Chris@0 959 QTextStream qts(&qs);
Chris@0 960 transform.toXml(qts);
Chris@0 961 cerr << qs.toStdString() << endl;
Chris@0 962 }