Chris@0: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@0: 
Chris@0: /*
Chris@0:     Sonic Annotator
Chris@0:     A utility for batch feature extraction from audio files.
Chris@0:     Mark Levy, Chris Sutton and Chris Cannam, Queen Mary, University of London.
Chris@0:     Copyright 2007-2008 QMUL.
Chris@0: 
Chris@0:     This program is free software; you can redistribute it and/or
Chris@0:     modify it under the terms of the GNU General Public License as
Chris@0:     published by the Free Software Foundation; either version 2 of the
Chris@0:     License, or (at your option) any later version.  See the file
Chris@0:     COPYING included with this distribution for more information.
Chris@0: */
Chris@0: 
Chris@0: #include "FeatureExtractionManager.h"
Chris@0: 
Chris@0: #include <vamp-hostsdk/PluginChannelAdapter.h>
Chris@0: #include <vamp-hostsdk/PluginBufferingAdapter.h>
Chris@0: #include <vamp-hostsdk/PluginInputDomainAdapter.h>
Chris@0: #include <vamp-hostsdk/PluginSummarisingAdapter.h>
Chris@8: #include <vamp-hostsdk/PluginWrapper.h>
Chris@0: #include <vamp-hostsdk/PluginLoader.h>
Chris@0: 
Chris@21: #include "base/Exceptions.h"
Chris@21: 
Chris@0: #include <iostream>
Chris@0: 
Chris@0: using namespace std;
Chris@0: 
Chris@0: using Vamp::Plugin;
Chris@0: using Vamp::PluginBase;
Chris@0: using Vamp::HostExt::PluginLoader;
Chris@0: using Vamp::HostExt::PluginChannelAdapter;
Chris@0: using Vamp::HostExt::PluginBufferingAdapter;
Chris@0: using Vamp::HostExt::PluginInputDomainAdapter;
Chris@0: using Vamp::HostExt::PluginSummarisingAdapter;
Chris@8: using Vamp::HostExt::PluginWrapper;
Chris@0: 
Chris@0: #include "data/fileio/FileSource.h"
Chris@0: #include "data/fileio/AudioFileReader.h"
Chris@0: #include "data/fileio/AudioFileReaderFactory.h"
Chris@0: #include "data/fileio/PlaylistFileReader.h"
Chris@0: #include "base/TempDirectory.h"
Chris@0: #include "base/ProgressPrinter.h"
Chris@0: #include "transform/TransformFactory.h"
Chris@0: #include "rdf/RDFTransformFactory.h"
Chris@0: #include "transform/FeatureWriter.h"
Chris@0: 
Chris@0: #include <QTextStream>
Chris@0: #include <QFile>
Chris@0: #include <QFileInfo>
Chris@0: 
Chris@0: FeatureExtractionManager::FeatureExtractionManager() :
Chris@0:     m_summariesOnly(false),
Chris@0:     // We can read using an arbitrary fixed block size --
Chris@0:     // PluginBufferingAdapter handles this for us.  It's likely to be
Chris@0:     // quicker to use larger sizes than smallish ones like 1024
Chris@0:     m_blockSize(16384),
Chris@0:     m_defaultSampleRate(0),
Chris@0:     m_sampleRate(0),
Chris@0:     m_channels(1)
Chris@0: {
Chris@0: }
Chris@0: 
Chris@0: FeatureExtractionManager::~FeatureExtractionManager()
Chris@0: {
Chris@0:     for (PluginMap::iterator pi = m_plugins.begin();
Chris@0:          pi != m_plugins.end(); ++pi) {
Chris@0:         delete pi->first;
Chris@0:     }
Chris@0: }
Chris@0: 
Chris@0: void FeatureExtractionManager::setChannels(int channels)
Chris@0: {
Chris@0:     m_channels = channels;
Chris@0: }
Chris@0: 
Chris@0: void FeatureExtractionManager::setDefaultSampleRate(int sampleRate)
Chris@0: {
Chris@0:     m_defaultSampleRate = sampleRate;
Chris@0: }
Chris@0: 
Chris@0: static PluginSummarisingAdapter::SummaryType
Chris@0: getSummaryType(string name)
Chris@0: {
Chris@0:     if (name == "min")      return PluginSummarisingAdapter::Minimum;
Chris@0:     if (name == "max")      return PluginSummarisingAdapter::Maximum;
Chris@0:     if (name == "mean")     return PluginSummarisingAdapter::Mean;
Chris@0:     if (name == "median")   return PluginSummarisingAdapter::Median;
Chris@0:     if (name == "mode")     return PluginSummarisingAdapter::Mode;
Chris@0:     if (name == "sum")      return PluginSummarisingAdapter::Sum;
Chris@0:     if (name == "variance") return PluginSummarisingAdapter::Variance;
Chris@0:     if (name == "sd")       return PluginSummarisingAdapter::StandardDeviation;
Chris@0:     if (name == "count")    return PluginSummarisingAdapter::Count;
Chris@0:     return PluginSummarisingAdapter::UnknownSummaryType;
Chris@0: }
Chris@0: 
Chris@0: bool FeatureExtractionManager::setSummaryTypes(const set<string> &names,
Chris@0:                                                bool summariesOnly,
Chris@0:                                                const PluginSummarisingAdapter::SegmentBoundaries &boundaries)
Chris@0: {
Chris@0:     for (SummaryNameSet::const_iterator i = names.begin();
Chris@0:          i != names.end(); ++i) {
Chris@0:         if (getSummaryType(*i) == PluginSummarisingAdapter::UnknownSummaryType) {
Chris@0:             cerr << "ERROR: Unknown summary type \"" << *i << "\"" << endl;
Chris@0:             return false;
Chris@0:         }
Chris@0:     }
Chris@0:     m_summaries = names;
Chris@0:     m_summariesOnly = summariesOnly;
Chris@0:     m_boundaries = boundaries;
Chris@0:     return true;
Chris@0: }
Chris@0: 
Chris@0: bool FeatureExtractionManager::addFeatureExtractor
Chris@0: (Transform transform, const vector<FeatureWriter*> &writers)
Chris@0: {
Chris@0:     //!!! exceptions rather than return values?
Chris@0: 
Chris@0:     if (transform.getSampleRate() == 0) {
Chris@0:         if (m_sampleRate == 0) {
Chris@0:             cerr << "NOTE: Transform does not specify a sample rate, using default rate of " << m_defaultSampleRate << endl;
Chris@0:             transform.setSampleRate(m_defaultSampleRate);
Chris@0:             m_sampleRate = m_defaultSampleRate;
Chris@0:         } else {
Chris@0:             cerr << "NOTE: Transform does not specify a sample rate, using previous transform's rate of " << m_sampleRate << endl;
Chris@0:             transform.setSampleRate(m_sampleRate);
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     if (m_sampleRate == 0) {
Chris@0:         m_sampleRate = transform.getSampleRate();
Chris@0:     }
Chris@0: 
Chris@0:     if (transform.getSampleRate() != m_sampleRate) {
Chris@0:         cerr << "WARNING: Transform sample rate " << transform.getSampleRate() << " does not match previously specified transform rate of " << m_sampleRate << " -- only a single rate is supported for each run" << endl;
Chris@0:         cerr << "WARNING: Using previous rate of " << m_sampleRate << " for this transform as well" << endl;
Chris@0:         transform.setSampleRate(m_sampleRate);
Chris@0:     }
Chris@0: 
Chris@0:     Plugin *plugin = 0;
Chris@0: 
Chris@0:     // Remember what the original transform looked like, and index
Chris@0:     // based on this -- because we may be about to fill in the zeros
Chris@0:     // for step and block size, but we want any further copies with
Chris@0:     // the same zeros to match this one
Chris@0:     Transform originalTransform = transform;
Chris@0:     
Chris@0:     if (m_transformPluginMap.find(transform) == m_transformPluginMap.end()) {
Chris@0: 
Chris@0:         // Test whether we already have a transform that is identical
Chris@0:         // to this, except for the output requested and/or the summary
Chris@0:         // type -- if so, they should share plugin instances (a vital
Chris@0:         // optimisation)
Chris@0: 
Chris@0:         for (TransformPluginMap::iterator i = m_transformPluginMap.begin();
Chris@0:              i != m_transformPluginMap.end(); ++i) {
Chris@0:             Transform test = i->first;
Chris@0:             test.setOutput(transform.getOutput());
Chris@0:             test.setSummaryType(transform.getSummaryType());
Chris@0:             if (transform == test) {
Chris@0:                 cerr << "NOTE: Already have transform identical to this one (for \""
Chris@0:                      << transform.getIdentifier().toStdString()
Chris@0:                      << "\") in every detail except output identifier and/or "
Chris@0:                      << "summary type; sharing its plugin instance" << endl;
Chris@0:                 plugin = i->second;
Chris@0:                 if (transform.getSummaryType() != Transform::NoSummary &&
Chris@0:                     !dynamic_cast<PluginSummarisingAdapter *>(plugin)) {
Chris@0:                     plugin = new PluginSummarisingAdapter(plugin);
Chris@0:                     i->second = plugin;
Chris@0:                 }
Chris@0:                 break;
Chris@0:             }
Chris@0:         }
Chris@0: 
Chris@0:         if (!plugin) {
Chris@0: 
Chris@0:             TransformFactory *tf = TransformFactory::getInstance();
Chris@0: 
Chris@0:             PluginBase *pb = tf->instantiatePluginFor(transform);
Chris@0:             plugin = tf->downcastVampPlugin(pb);
Chris@0:             if (!plugin) {
Chris@0:                 //!!! todo: handle non-Vamp plugins too, or make the main --list
Chris@0:                 // option print out only Vamp transforms
Chris@0:                 cerr << "ERROR: Failed to load plugin for transform \""
Chris@0:                      << transform.getIdentifier().toStdString() << "\"" << endl;
Chris@0:                 delete pb;
Chris@0:                 return false;
Chris@0:             }
Chris@0:             
Chris@0:             // We will provide the plugin with arbitrary step and
Chris@0:             // block sizes (so that we can use the same read/write
Chris@0:             // block size for all transforms), and to that end we use
Chris@0:             // a PluginBufferingAdapter.  However, we need to know the
Chris@0:             // underlying step size so that we can provide the right
Chris@0:             // context for dense outputs.  (Although, don't forget
Chris@0:             // that the PluginBufferingAdapter rewrites
Chris@0:             // OneSamplePerStep outputs so as to use FixedSampleRate
Chris@0:             // -- so it supplies the sample rate in the output
Chris@0:             // feature.  I'm not sure whether we can easily use that.)
Chris@0: 
Chris@0:             size_t pluginStepSize = plugin->getPreferredStepSize();
Chris@0:             size_t pluginBlockSize = plugin->getPreferredBlockSize();
Chris@0: 
Chris@25:             PluginInputDomainAdapter *pida = 0;
Chris@25: 
Chris@0:             // adapt the plugin for buffering, channels, etc.
Chris@0:             if (plugin->getInputDomain() == Plugin::FrequencyDomain) {
Chris@25:                 pida = new PluginInputDomainAdapter(plugin);
Chris@26:                 pida->setProcessTimestampMethod(PluginInputDomainAdapter::ShiftData);
Chris@25:                 plugin = pida;
Chris@0:             }
Chris@0: 
Chris@0:             PluginBufferingAdapter *pba = new PluginBufferingAdapter(plugin);
Chris@0:             plugin = pba;
Chris@0: 
Chris@0:             if (transform.getStepSize() != 0) {
Chris@0:                 pba->setPluginStepSize(transform.getStepSize());
Chris@0:             } else {
Chris@0:                 transform.setStepSize(pluginStepSize);
Chris@0:             }
Chris@0: 
Chris@0:             if (transform.getBlockSize() != 0) {
Chris@0:                 pba->setPluginBlockSize(transform.getBlockSize());
Chris@0:             } else {
Chris@0:                 transform.setBlockSize(pluginBlockSize);
Chris@0:             }
Chris@0: 
Chris@0:             plugin = new PluginChannelAdapter(plugin);
Chris@0: 
Chris@0:             if (!m_summaries.empty() ||
Chris@0:                 transform.getSummaryType() != Transform::NoSummary) {
Chris@0:                 PluginSummarisingAdapter *adapter =
Chris@0:                     new PluginSummarisingAdapter(plugin);
Chris@0:                 adapter->setSummarySegmentBoundaries(m_boundaries);
Chris@0:                 plugin = adapter;
Chris@0:             }
Chris@0: 
Chris@0:             if (!plugin->initialise(m_channels, m_blockSize, m_blockSize)) {
Chris@0:                 cerr << "ERROR: Plugin initialise (channels = " << m_channels << ", stepSize = " << m_blockSize << ", blockSize = " << m_blockSize << ") failed." << endl;    
Chris@0:                 delete plugin;
Chris@0:                 return false;
Chris@0:             }
Chris@0: 
Chris@0: //            cerr << "Initialised plugin" << endl;
Chris@0: 
Chris@0:             size_t actualStepSize = 0;
Chris@0:             size_t actualBlockSize = 0;
Chris@0:             pba->getActualStepAndBlockSizes(actualStepSize, actualBlockSize);
Chris@0:             transform.setStepSize(actualStepSize);
Chris@0:             transform.setBlockSize(actualBlockSize);
Chris@0: 
Chris@0:             Plugin::OutputList outputs = plugin->getOutputDescriptors();
Chris@0:             for (int i = 0; i < (int)outputs.size(); ++i) {
Chris@0: 
Chris@0: //                cerr << "Newly initialised plugin output " << i << " has bin count " << outputs[i].binCount << endl;
Chris@0: 
Chris@0:                 m_pluginOutputs[plugin][outputs[i].identifier] = outputs[i];
Chris@0:                 m_pluginOutputIndices[outputs[i].identifier] = i;
Chris@0:             }
Chris@0: 
Chris@10:             cerr << "NOTE: Loaded and initialised plugin for transform \""
Chris@25:                  << transform.getIdentifier().toStdString()
Chris@25:                  << "\" with plugin step size " << actualStepSize
Chris@25:                  << " and block size " << actualBlockSize
Chris@25:                  << " (adapter step and block size " << m_blockSize << ")"
Chris@25:                  << endl;
Chris@25: 
Chris@25:             if (pida) {
Chris@25:                 cerr << "NOTE: PluginInputDomainAdapter timestamp adjustment is "
Chris@25: 
Chris@25:                      << pida->getTimestampAdjustment() << endl;
Chris@25:             }
Chris@8: 
Chris@8:         } else {
Chris@8: 
Chris@8:             if (transform.getStepSize() == 0 || transform.getBlockSize() == 0) {
Chris@8: 
Chris@8:                 PluginWrapper *pw = dynamic_cast<PluginWrapper *>(plugin);
Chris@8:                 if (pw) {
Chris@8:                     PluginBufferingAdapter *pba =
Chris@8:                         pw->getWrapper<PluginBufferingAdapter>();
Chris@8:                     if (pba) {
Chris@8:                         size_t actualStepSize = 0;
Chris@8:                         size_t actualBlockSize = 0;
Chris@8:                         pba->getActualStepAndBlockSizes(actualStepSize,
Chris@8:                                                         actualBlockSize);
Chris@8:                         if (transform.getStepSize() == 0) {
Chris@8:                             transform.setStepSize(actualStepSize);
Chris@8:                         }
Chris@8:                         if (transform.getBlockSize() == 0) {
Chris@8:                             transform.setBlockSize(actualBlockSize);
Chris@8:                         }
Chris@8:                     }
Chris@8:                 }
Chris@8:             }
Chris@0:         }
Chris@0: 
Chris@0:         if (transform.getOutput() == "") {
Chris@0:             transform.setOutput
Chris@0:                 (plugin->getOutputDescriptors()[0].identifier.c_str());
Chris@0:         }
Chris@0: 
Chris@0:         m_transformPluginMap[transform] = plugin;
Chris@0: 
Chris@0:         if (!(originalTransform == transform)) {
Chris@0:             m_transformPluginMap[originalTransform] = plugin;
Chris@0:         }
Chris@0: 
Chris@0:     } else {
Chris@0:         
Chris@0:         plugin = m_transformPluginMap[transform];
Chris@0:     }
Chris@0: 
Chris@0:     m_plugins[plugin][transform] = writers;
Chris@0: 
Chris@0:     return true;
Chris@0: }
Chris@0: 
Chris@0: bool FeatureExtractionManager::addDefaultFeatureExtractor
Chris@0: (TransformId transformId, const vector<FeatureWriter*> &writers)
Chris@0: {
Chris@0:     TransformFactory *tf = TransformFactory::getInstance();
Chris@0: 
Chris@0:     if (m_sampleRate == 0) {
Chris@0:         if (m_defaultSampleRate == 0) {
Chris@0:             cerr << "ERROR: Default transform requested, but no default sample rate available" << endl;
Chris@0:             return false;
Chris@0:         } else {
Chris@0:             cerr << "NOTE: Using default sample rate of " << m_defaultSampleRate << " for default transform" << endl;
Chris@0:             m_sampleRate = m_defaultSampleRate;
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     Transform transform = tf->getDefaultTransformFor(transformId, m_sampleRate);
Chris@0: 
Chris@0:     return addFeatureExtractor(transform, writers);
Chris@0: }
Chris@0: 
Chris@0: bool FeatureExtractionManager::addFeatureExtractorFromFile
Chris@0: (QString transformXmlFile, const vector<FeatureWriter*> &writers)
Chris@0: {
Chris@0:     RDFTransformFactory factory
Chris@0:         (QUrl::fromLocalFile(QFileInfo(transformXmlFile).absoluteFilePath())
Chris@0:          .toString());
Chris@0:     ProgressPrinter printer("Parsing transforms RDF file");
Chris@0:     std::vector<Transform> transforms = factory.getTransforms(&printer);
Chris@0:     if (!factory.isOK()) {
Chris@0:         cerr << "WARNING: FeatureExtractionManager::addFeatureExtractorFromFile: Failed to parse transforms file: " << factory.getErrorString().toStdString() << endl;
Chris@0:         if (factory.isRDF()) {
Chris@0:             return false; // no point trying it as XML
Chris@0:         }
Chris@0:     }
Chris@0:     if (!transforms.empty()) {
Chris@0:         bool success = true;
Chris@0:         for (int i = 0; i < (int)transforms.size(); ++i) {
Chris@0:             if (!addFeatureExtractor(transforms[i], writers)) {
Chris@0:                 success = false;
Chris@0:             }
Chris@0:         }
Chris@0:         return success;
Chris@0:     }
Chris@0: 
Chris@0:     QFile file(transformXmlFile);
Chris@0:     if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
Chris@0:         cerr << "ERROR: Failed to open transform XML file \""
Chris@0:              << transformXmlFile.toStdString() << "\" for reading" << endl;
Chris@0:         return false;
Chris@0:     }
Chris@0: 
Chris@0:     QTextStream *qts = new QTextStream(&file);
Chris@0:     QString qs = qts->readAll();
Chris@0:     delete qts;
Chris@0:     file.close();
Chris@0: 
Chris@0:     Transform transform(qs);
Chris@0: 
Chris@0:     return addFeatureExtractor(transform, writers);
Chris@0: }
Chris@0: 
Chris@0: void FeatureExtractionManager::extractFeatures(QString audioSource)
Chris@0: {
Chris@0:     if (m_plugins.empty()) return;
Chris@0: 
Chris@11:     ProgressPrinter retrievalProgress("Retrieving audio data...");
Chris@0: 
Chris@11:     FileSource source(audioSource, &retrievalProgress);
Chris@0:     if (!source.isAvailable()) {
Chris@0:         cerr << "ERROR: File or URL \"" << audioSource.toStdString()
Chris@0:              << "\" could not be located" << endl;
Chris@21:         throw FileNotFound(audioSource);
Chris@0:     }
Chris@0:     
Chris@0:     source.waitForData();
Chris@0:     
Chris@0:     if (QFileInfo(audioSource).suffix().toLower() == "m3u") {
Chris@0:         PlaylistFileReader reader(source);
Chris@0:         if (reader.isOK()) {
Chris@0:             vector<QString> files = reader.load();
Chris@0:             for (int i = 0; i < (int)files.size(); ++i) {
Chris@0:                 extractFeatures(files[i]);
Chris@0:             }
Chris@0:             return;
Chris@0:         } else {
Chris@0:             cerr << "ERROR: Playlist \"" << audioSource.toStdString()
Chris@0:                  << "\" could not be opened" << endl;
Chris@21:             throw FileNotFound(audioSource);
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     if (m_sampleRate == 0) {
Chris@0:         cerr << "ERROR: Internal error in FeatureExtractionManager::extractFeatures: Plugin list is non-empty, but no sample rate set" << endl;
Chris@0:         exit(1);
Chris@0:     }
Chris@0: 
Chris@0:     AudioFileReader *reader =
Chris@11:         AudioFileReaderFactory::createReader(source, m_sampleRate, &retrievalProgress);
Chris@0:     
Chris@0:     if (!reader) {
Chris@21:         throw FailedToOpenFile(audioSource);
Chris@0:     }
Chris@0: 
Chris@0:     size_t channels = reader->getChannelCount();
Chris@11: 
Chris@11:     retrievalProgress.done();
Chris@11: 
Chris@0:     cerr << "Opened " << channels << "-channel file or URL \"" << audioSource.toStdString() << "\"" << endl;
Chris@0: 
Chris@0:     // reject file if it has too few channels, plugin will handle if it has too many
Chris@0:     if ((int)channels < m_channels) {
Chris@21:         throw FileOperationFailed
Chris@21:             (audioSource,
Chris@21:              QString("read sufficient channels (found %1, require %2)")
Chris@21:              .arg(channels).arg(m_channels));
Chris@0:     }
Chris@0:     
Chris@0:     // allocate audio buffers
Chris@0:     float **data = new float *[m_channels];
Chris@0:     for (int c = 0; c < m_channels; ++c) {
Chris@0:         data[c] = new float[m_blockSize];
Chris@0:     }
Chris@0: 
Chris@0:     size_t frameCount = reader->getFrameCount();
Chris@0:     
Chris@0: //    cerr << "file has " << frameCount << " frames" << endl;
Chris@0: 
Chris@0:     for (PluginMap::iterator pi = m_plugins.begin();
Chris@0:          pi != m_plugins.end(); ++pi) {
Chris@0: 
Chris@0:         Plugin *plugin = pi->first;
Chris@0: 
Chris@0: //        std::cerr << "Calling reset on " << plugin << std::endl;
Chris@0:         plugin->reset();
Chris@0: 
Chris@0:         for (TransformWriterMap::iterator ti = pi->second.begin();
Chris@0:              ti != pi->second.end(); ++ti) {
Chris@0: 
Chris@0:             const Transform &transform = ti->first;
Chris@0: 
Chris@0:             //!!! we may want to set the start and duration times for extraction
Chris@0:             // in the transform record (defaults of zero indicate extraction
Chris@0:             // from the whole file)
Chris@0: //            transform.setStartTime(RealTime::zeroTime);
Chris@0: //            transform.setDuration
Chris@0: //                (RealTime::frame2RealTime(reader->getFrameCount(), m_sampleRate));
Chris@0: 
Chris@0:             string outputId = transform.getOutput().toStdString();
Chris@0:             if (m_pluginOutputs[plugin].find(outputId) ==
Chris@0:                 m_pluginOutputs[plugin].end()) {
Chris@0:                 //!!! throw?
Chris@0:                 cerr << "WARNING: Nonexistent plugin output \"" << outputId << "\" requested for transform \""
Chris@0:                      << transform.getIdentifier().toStdString() << "\", ignoring this transform"
Chris@0:                      << endl;
Chris@0: /*
Chris@0:                 cerr << "Known outputs for all plugins are as follows:" << endl;
Chris@0:                 for (PluginOutputMap::const_iterator k = m_pluginOutputs.begin();
Chris@0:                      k != m_pluginOutputs.end(); ++k) {
Chris@0:                     cerr << "Plugin " << k->first << ": ";
Chris@0:                     if (k->second.empty()) {
Chris@0:                         cerr << "(none)";
Chris@0:                     }
Chris@0:                     for (OutputMap::const_iterator i = k->second.begin();
Chris@0:                          i != k->second.end(); ++i) {
Chris@0:                         cerr << "\"" << i->first << "\" ";
Chris@0:                     }
Chris@0:                     cerr << endl;
Chris@0:                 }
Chris@0: */
Chris@0:             }
Chris@0:         }
Chris@0:     }
Chris@0:     
Chris@0:     long startFrame = 0;
Chris@0:     long endFrame = frameCount;
Chris@0: 
Chris@0: /*!!! No -- there is no single transform to pull this stuff from --
Chris@0:  * the transforms may have various start and end times, need to be far
Chris@0:  * cleverer about this if we're going to support them
Chris@0: 
Chris@0:     RealTime trStartRT = transform.getStartTime();
Chris@0:     RealTime trDurationRT = transform.getDuration();
Chris@0: 
Chris@0:     long trStart = RealTime::realTime2Frame(trStartRT, m_sampleRate);
Chris@0:     long trDuration = RealTime::realTime2Frame(trDurationRT, m_sampleRate);
Chris@0: 
Chris@0:     if (trStart == 0 || trStart < startFrame) {
Chris@0:         trStart = startFrame;
Chris@0:     }
Chris@0: 
Chris@0:     if (trDuration == 0) {
Chris@0:         trDuration = endFrame - trStart;
Chris@0:     }
Chris@0:     if (trStart + trDuration > endFrame) {
Chris@0:         trDuration = endFrame - trStart;
Chris@0:     }
Chris@0: 
Chris@0:     startFrame = trStart;
Chris@0:     endFrame = trStart + trDuration;
Chris@0: */
Chris@0:     
Chris@0:     for (PluginMap::iterator pi = m_plugins.begin();
Chris@0:          pi != m_plugins.end(); ++pi) { 
Chris@0: 
Chris@0:         for (TransformWriterMap::const_iterator ti = pi->second.begin();
Chris@0:              ti != pi->second.end(); ++ti) {
Chris@0:         
Chris@0:             const vector<FeatureWriter *> &writers = ti->second;
Chris@0:             
Chris@0:             for (int j = 0; j < (int)writers.size(); ++j) {
Chris@0:                 FeatureWriter::TrackMetadata m;
Chris@0:                 m.title = reader->getTitle();
Chris@0:                 m.maker = reader->getMaker();
Chris@19:                 if (m.title != "" && m.maker != "") {
Chris@19:                     writers[j]->setTrackMetadata(audioSource, m);
Chris@19:                 }
Chris@0:             }
Chris@0:         }
Chris@0:     }
Chris@0: 
Chris@0:     ProgressPrinter extractionProgress("Extracting and writing features...");
Chris@0:     int progress = 0;
Chris@0: 
Chris@0:     for (long i = startFrame; i < endFrame; i += m_blockSize) {
Chris@0:         
Chris@0:         //!!! inefficient, although much of the inefficiency may be
Chris@0:         // susceptible to optimisation
Chris@0:         
Chris@0:         SampleBlock frames;
Chris@0:         reader->getInterleavedFrames(i, m_blockSize, frames);
Chris@0:         
Chris@0:         // We have to do our own channel handling here; we can't just
Chris@0:         // leave it to the plugin adapter because the same plugin
Chris@0:         // adapter may have to serve for input files with various
Chris@0:         // numbers of channels (so the adapter is simply configured
Chris@0:         // with a fixed channel count, generally 1).
Chris@0: 
Chris@0:         int rc = reader->getChannelCount();
Chris@0: 
Chris@0:         for (int j = 0; j < m_blockSize; ++j) {
Chris@0:             for (int c = 0; c < m_channels; ++c) {
Chris@0:                 int index;
Chris@0:                 if (c < rc) {
Chris@0:                     index = j * rc + c;
Chris@0:                     data[c][j] = 0.f;
Chris@0:                 } else {
Chris@0:                     index = j * rc + (c % rc);
Chris@0:                 }
Chris@0:                 if (index < (int)frames.size()) {
Chris@0:                     data[c][j] += frames[index];
Chris@0:                 }
Chris@0:             }
Chris@0:         }    
Chris@0: 
Chris@0:         Vamp::RealTime timestamp = Vamp::RealTime::frame2RealTime
Chris@0:             (i, m_sampleRate);
Chris@0:         
Chris@0:         for (PluginMap::iterator pi = m_plugins.begin();
Chris@0:              pi != m_plugins.end(); ++pi) {
Chris@0: 
Chris@0:             Plugin *plugin = pi->first;
Chris@0:             Plugin::FeatureSet featureSet = plugin->process(data, timestamp);
Chris@0: 
Chris@0:             if (!m_summariesOnly) {
Chris@0:                 writeFeatures(audioSource, plugin, featureSet);
Chris@0:             }
Chris@0:         }
Chris@0: 
Chris@0:         int pp = progress;
Chris@6:         progress = int(((i - startFrame) * 100.0) / (endFrame - startFrame) + 0.1);
Chris@0:         if (progress > pp) extractionProgress.setProgress(progress);
Chris@0:     }
Chris@10: 
Chris@22: //    std::cerr << "FeatureExtractionManager: deleting audio file reader" << std::endl;
Chris@12: 
Chris@10:     delete reader;
Chris@0:     
Chris@0:     for (PluginMap::iterator pi = m_plugins.begin();
Chris@0:          pi != m_plugins.end(); ++pi) { 
Chris@0: 
Chris@0:         Plugin *plugin = pi->first;
Chris@0:         Plugin::FeatureSet featureSet = plugin->getRemainingFeatures();
Chris@0: 
Chris@0:         if (!m_summariesOnly) {
Chris@0:             writeFeatures(audioSource, plugin, featureSet);
Chris@0:         }
Chris@0: 
Chris@0:         if (!m_summaries.empty()) {
Chris@0:             PluginSummarisingAdapter *adapter =
Chris@0:                 dynamic_cast<PluginSummarisingAdapter *>(plugin);
Chris@0:             if (!adapter) {
Chris@0:                 cerr << "WARNING: Summaries requested, but plugin is not a summarising adapter" << endl;
Chris@0:             } else {
Chris@0:                 for (SummaryNameSet::const_iterator sni = m_summaries.begin();
Chris@0:                      sni != m_summaries.end(); ++sni) {
Chris@0:                     featureSet.clear();
Chris@0:                     //!!! problem here -- we are requesting summaries
Chris@0:                     //!!! for all outputs, but they in principle have
Chris@0:                     //!!! different averaging requirements depending
Chris@0:                     //!!! on whether their features have duration or
Chris@0:                     //!!! not
Chris@0:                     featureSet = adapter->getSummaryForAllOutputs
Chris@0:                         (getSummaryType(*sni),
Chris@0:                          PluginSummarisingAdapter::ContinuousTimeAverage);
Chris@0:                     writeFeatures(audioSource, plugin, featureSet,//!!! *sni);
Chris@0:                                   Transform::stringToSummaryType(sni->c_str()));
Chris@0:                 }
Chris@0:             }
Chris@0:         }
Chris@0: 
Chris@0:         writeSummaries(audioSource, plugin);
Chris@0:     }
Chris@0: 
Chris@3:     extractionProgress.done();
Chris@3: 
Chris@0:     finish();
Chris@0:     
Chris@0:     TempDirectory::getInstance()->cleanup();
Chris@0: }
Chris@0: 
Chris@0: void
Chris@0: FeatureExtractionManager::writeSummaries(QString audioSource, Plugin *plugin)
Chris@0: {
Chris@0:     // caller should have ensured plugin is in m_plugins
Chris@0:     PluginMap::iterator pi = m_plugins.find(plugin);
Chris@0: 
Chris@0:     for (TransformWriterMap::const_iterator ti = pi->second.begin();
Chris@0:          ti != pi->second.end(); ++ti) {
Chris@0:         
Chris@0:         const Transform &transform = ti->first;
Chris@0:         const vector<FeatureWriter *> &writers = ti->second;
Chris@0: 
Chris@0:         Transform::SummaryType summaryType = transform.getSummaryType();
Chris@0:         PluginSummarisingAdapter::SummaryType pType =
Chris@0:             (PluginSummarisingAdapter::SummaryType)summaryType;
Chris@0: 
Chris@0:         if (transform.getSummaryType() == Transform::NoSummary) {
Chris@0:             continue;
Chris@0:         }
Chris@0: 
Chris@0:         PluginSummarisingAdapter *adapter =
Chris@0:             dynamic_cast<PluginSummarisingAdapter *>(plugin);
Chris@0:         if (!adapter) {
Chris@0:             cerr << "FeatureExtractionManager::writeSummaries: INTERNAL ERROR: Summary requested for transform, but plugin is not a summarising adapter" << endl;
Chris@0:             continue;
Chris@0:         }
Chris@0: 
Chris@0:         Plugin::FeatureSet featureSet = adapter->getSummaryForAllOutputs
Chris@0:             (pType, PluginSummarisingAdapter::ContinuousTimeAverage);
Chris@0: 
Chris@0: //        cout << "summary type " << int(pType) << " for transform:" << endl << transform.toXmlString().toStdString()<< endl << "... feature set with " << featureSet.size() << " elts" << endl;
Chris@0: 
Chris@0:         writeFeatures(audioSource, plugin, featureSet, summaryType);
Chris@0:     }
Chris@0: }
Chris@0: 
Chris@0: void FeatureExtractionManager::writeFeatures(QString audioSource,
Chris@0:                                              Plugin *plugin,
Chris@0:                                              const Plugin::FeatureSet &features,
Chris@0:                                              Transform::SummaryType summaryType)
Chris@0: {
Chris@0:     // caller should have ensured plugin is in m_plugins
Chris@0:     PluginMap::iterator pi = m_plugins.find(plugin);
Chris@0: 
Chris@0:     for (TransformWriterMap::const_iterator ti = pi->second.begin();
Chris@0:          ti != pi->second.end(); ++ti) {
Chris@0:         
Chris@0:         const Transform &transform = ti->first;
Chris@0:         const vector<FeatureWriter *> &writers = ti->second;
Chris@0:         
Chris@0:         if (transform.getSummaryType() != Transform::NoSummary &&
Chris@0:             m_summaries.empty() &&
Chris@0:             summaryType == Transform::NoSummary) {
Chris@0:             continue;
Chris@0:         }
Chris@0: 
Chris@0:         if (transform.getSummaryType() != Transform::NoSummary &&
Chris@0:             summaryType != Transform::NoSummary &&
Chris@0:             transform.getSummaryType() != summaryType) {
Chris@0:             continue;
Chris@0:         }
Chris@0: 
Chris@0:         string outputId = transform.getOutput().toStdString();
Chris@0: 
Chris@0:         if (m_pluginOutputs[plugin].find(outputId) ==
Chris@0:             m_pluginOutputs[plugin].end()) {
Chris@0:             continue;
Chris@0:         }
Chris@0:         
Chris@0:         const Plugin::OutputDescriptor &desc =
Chris@0:             m_pluginOutputs[plugin][outputId];
Chris@0:         
Chris@0:         int outputIndex = m_pluginOutputIndices[outputId];
Chris@0:         Plugin::FeatureSet::const_iterator fsi = features.find(outputIndex);
Chris@0:         if (fsi == features.end()) continue;
Chris@0: 
Chris@0:         for (int j = 0; j < (int)writers.size(); ++j) {
Chris@0:             writers[j]->write
Chris@0:                 (audioSource, transform, desc, fsi->second,
Chris@0:                  Transform::summaryTypeToString(summaryType).toStdString());
Chris@0:         }
Chris@0:     }
Chris@0: }
Chris@0: 
Chris@0: void FeatureExtractionManager::finish()
Chris@0: {
Chris@0:     for (PluginMap::iterator pi = m_plugins.begin();
Chris@0:          pi != m_plugins.end(); ++pi) {
Chris@0: 
Chris@0:         for (TransformWriterMap::iterator ti = pi->second.begin();
Chris@0:              ti != pi->second.end(); ++ti) {
Chris@0:         
Chris@0:             vector<FeatureWriter *> &writers = ti->second;
Chris@0: 
Chris@0:             for (int i = 0; i < (int)writers.size(); ++i) {
Chris@0:                 writers[i]->flush();
Chris@0:                 writers[i]->finish();
Chris@0:             }
Chris@0:         }
Chris@0:     }
Chris@0: }
Chris@0: 
Chris@0: void FeatureExtractionManager::print(Transform transform) const
Chris@0: {
Chris@0:     QString qs;
Chris@0:     QTextStream qts(&qs);
Chris@0:     transform.toXml(qts);
Chris@0:     cerr << qs.toStdString() << endl;
Chris@0: }