cannam@173: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
cannam@173: 
cannam@173: /*
cannam@173:     Vamp
cannam@173: 
cannam@173:     An API for audio analysis and feature extraction plugins.
cannam@173: 
cannam@173:     Centre for Digital Music, Queen Mary, University of London.
cannam@173:     Copyright 2006-2008 Chris Cannam and QMUL.
cannam@173:   
cannam@173:     Permission is hereby granted, free of charge, to any person
cannam@173:     obtaining a copy of this software and associated documentation
cannam@173:     files (the "Software"), to deal in the Software without
cannam@173:     restriction, including without limitation the rights to use, copy,
cannam@173:     modify, merge, publish, distribute, sublicense, and/or sell copies
cannam@173:     of the Software, and to permit persons to whom the Software is
cannam@173:     furnished to do so, subject to the following conditions:
cannam@173: 
cannam@173:     The above copyright notice and this permission notice shall be
cannam@173:     included in all copies or substantial portions of the Software.
cannam@173: 
cannam@173:     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
cannam@173:     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
cannam@173:     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
cannam@173:     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
cannam@173:     ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
cannam@173:     CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
cannam@173:     WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
cannam@173: 
cannam@173:     Except as contained in this notice, the names of the Centre for
cannam@173:     Digital Music; Queen Mary, University of London; and Chris Cannam
cannam@173:     shall not be used in advertising or otherwise to promote the sale,
cannam@173:     use or other dealings in this Software without prior written
cannam@173:     authorization.
cannam@173: */
cannam@173: 
cannam@173: #include "PluginSummarisingAdapter.h"
cannam@173: 
cannam@174: #include <map>
cannam@175: #include <cmath>
cannam@174: 
cannam@173: namespace Vamp {
cannam@173: 
cannam@173: namespace HostExt {
cannam@173: 
cannam@173: class PluginSummarisingAdapter::Impl
cannam@173: {
cannam@173: public:
cannam@173:     Impl(Plugin *plugin, float inputSampleRate);
cannam@173:     ~Impl();
cannam@173: 
cannam@173:     FeatureSet process(const float *const *inputBuffers, RealTime timestamp);
cannam@173:     FeatureSet getRemainingFeatures();
cannam@173: 
cannam@173:     void setSummarySegmentBoundaries(const SegmentBoundaries &);
cannam@173: 
cannam@180:     FeatureList getSummaryForOutput(int output,
cannam@180:                                     SummaryType type,
cannam@180:                                     AveragingMethod avg);
cannam@180: 
cannam@180:     FeatureSet getSummaryForAllOutputs(SummaryType type,
cannam@180:                                        AveragingMethod avg);
cannam@173: 
cannam@173: protected:
cannam@174:     Plugin *m_plugin;
cannam@181:     float m_inputSampleRate;
cannam@174: 
cannam@173:     SegmentBoundaries m_boundaries;
cannam@174:     
cannam@174:     typedef std::vector<float> ValueList;
cannam@174:     typedef std::map<int, ValueList> BinValueMap;
cannam@180:     typedef std::vector<RealTime> DurationList;
cannam@174:     
cannam@174:     struct OutputAccumulator {
cannam@174:         int count;
cannam@180:         BinValueMap values; // bin number -> values ordered by time
cannam@180:         DurationList durations;
cannam@180:         OutputAccumulator() : count(0), values(), durations() { }
cannam@174:     };
cannam@174: 
cannam@174:     typedef std::map<int, OutputAccumulator> OutputAccumulatorMap;
cannam@180:     OutputAccumulatorMap m_accumulators; // output number -> accumulator
cannam@180: 
cannam@180:     typedef std::map<int, RealTime> OutputTimestampMap;
cannam@180:     OutputTimestampMap m_prevTimestamps; // output number -> timestamp
cannam@183:     OutputTimestampMap m_prevDurations; // output number -> durations
cannam@174: 
cannam@174:     struct OutputBinSummary {
cannam@180: 
cannam@180:         int count;
cannam@180: 
cannam@180:         // extents
cannam@174:         float minimum;
cannam@174:         float maximum;
cannam@180:         float sum;
cannam@180: 
cannam@180:         // sample-average results
cannam@174:         float median;
cannam@174:         float mode;
cannam@174:         float variance;
cannam@180: 
cannam@180:         // continuous-time average results
cannam@180:         float median_c;
cannam@180:         float mode_c;
cannam@180:         float mean_c;
cannam@180:         float variance_c;
cannam@174:     };
cannam@174: 
cannam@174:     typedef std::map<int, OutputBinSummary> OutputSummary;
cannam@174:     typedef std::map<RealTime, OutputSummary> SummarySegmentMap;
cannam@174:     typedef std::map<int, SummarySegmentMap> OutputSummarySegmentMap;
cannam@174: 
cannam@174:     OutputSummarySegmentMap m_summaries;
cannam@174: 
cannam@183:     bool m_reduced;
cannam@174:     RealTime m_lastTimestamp;
cannam@174: 
cannam@180:     void accumulate(const FeatureSet &fs, RealTime, bool final);
cannam@180:     void accumulate(int output, const Feature &f, RealTime, bool final);
cannam@174:     void reduce();
cannam@173: };
cannam@173:     
cannam@173: PluginSummarisingAdapter::PluginSummarisingAdapter(Plugin *plugin) :
cannam@173:     PluginWrapper(plugin)
cannam@173: {
cannam@173:     m_impl = new Impl(plugin, m_inputSampleRate);
cannam@173: }
cannam@173: 
cannam@173: PluginSummarisingAdapter::~PluginSummarisingAdapter()
cannam@173: {
cannam@173:     delete m_impl;
cannam@173: }
cannam@173: 
cannam@173: Plugin::FeatureSet
cannam@173: PluginSummarisingAdapter::process(const float *const *inputBuffers, RealTime timestamp)
cannam@173: {
cannam@173:     return m_impl->process(inputBuffers, timestamp);
cannam@173: }
cannam@173: 
cannam@174: Plugin::FeatureSet
cannam@174: PluginSummarisingAdapter::getRemainingFeatures()
cannam@174: {
cannam@174:     return m_impl->getRemainingFeatures();
cannam@174: }
cannam@174: 
cannam@175: Plugin::FeatureList
cannam@180: PluginSummarisingAdapter::getSummaryForOutput(int output,
cannam@180:                                               SummaryType type,
cannam@180:                                               AveragingMethod avg)
cannam@175: {
cannam@180:     return m_impl->getSummaryForOutput(output, type, avg);
cannam@176: }
cannam@176: 
cannam@176: Plugin::FeatureSet
cannam@180: PluginSummarisingAdapter::getSummaryForAllOutputs(SummaryType type,
cannam@180:                                                   AveragingMethod avg)
cannam@176: {
cannam@180:     return m_impl->getSummaryForAllOutputs(type, avg);
cannam@175: }
cannam@173: 
cannam@173: PluginSummarisingAdapter::Impl::Impl(Plugin *plugin, float inputSampleRate) :
cannam@181:     m_plugin(plugin),
cannam@183:     m_inputSampleRate(inputSampleRate),
cannam@183:     m_reduced(false)
cannam@173: {
cannam@173: }
cannam@173: 
cannam@173: PluginSummarisingAdapter::Impl::~Impl()
cannam@173: {
cannam@173: }
cannam@173: 
cannam@174: Plugin::FeatureSet
cannam@174: PluginSummarisingAdapter::Impl::process(const float *const *inputBuffers, RealTime timestamp)
cannam@174: {
cannam@183:     if (m_reduced) {
cannam@183:         std::cerr << "WARNING: Cannot call PluginSummarisingAdapter::process() or getRemainingFeatures() after one of the getSummary methods" << std::endl;
cannam@183:     }
cannam@174:     FeatureSet fs = m_plugin->process(inputBuffers, timestamp);
cannam@180:     accumulate(fs, timestamp, false);
cannam@174:     m_lastTimestamp = timestamp;
cannam@174:     return fs;
cannam@174: }
cannam@174: 
cannam@174: Plugin::FeatureSet
cannam@174: PluginSummarisingAdapter::Impl::getRemainingFeatures()
cannam@174: {
cannam@183:     if (m_reduced) {
cannam@183:         std::cerr << "WARNING: Cannot call PluginSummarisingAdapter::process() or getRemainingFeatures() after one of the getSummary methods" << std::endl;
cannam@183:     }
cannam@174:     FeatureSet fs = m_plugin->getRemainingFeatures();
cannam@180:     accumulate(fs, m_lastTimestamp, true);
cannam@174:     return fs;
cannam@174: }
cannam@174: 
cannam@175: Plugin::FeatureList
cannam@180: PluginSummarisingAdapter::Impl::getSummaryForOutput(int output,
cannam@180:                                                     SummaryType type,
cannam@180:                                                     AveragingMethod avg)
cannam@175: {
cannam@183:     if (!m_reduced) reduce();
cannam@183: 
cannam@180:     bool continuous = (avg == ContinuousTimeAverage);
cannam@180: 
cannam@175:     FeatureList fl;
cannam@175:     for (SummarySegmentMap::const_iterator i = m_summaries[output].begin();
cannam@175:          i != m_summaries[output].end(); ++i) {
cannam@177: 
cannam@175:         Feature f;
cannam@175:         f.hasTimestamp = true;
cannam@175:         f.timestamp = i->first;
cannam@175:         f.hasDuration = false;
cannam@177: 
cannam@175:         for (OutputSummary::const_iterator j = i->second.begin();
cannam@175:              j != i->second.end(); ++j) {
cannam@175: 
cannam@175:             // these will be ordered by bin number, and no bin numbers
cannam@175:             // will be missing except at the end (because of the way
cannam@175:             // the accumulators were initially filled in accumulate())
cannam@175: 
cannam@175:             const OutputBinSummary &summary = j->second;
cannam@175:             float result = 0.f;
cannam@175: 
cannam@175:             switch (type) {
cannam@175: 
cannam@175:             case Minimum:
cannam@175:                 result = summary.minimum;
cannam@175:                 break;
cannam@175: 
cannam@175:             case Maximum:
cannam@175:                 result = summary.maximum;
cannam@175:                 break;
cannam@175: 
cannam@175:             case Mean:
cannam@180:                 if (continuous) {
cannam@180:                     result = summary.mean_c;
cannam@180:                 } else if (summary.count) {
cannam@175:                     result = summary.sum / summary.count;
cannam@175:                 }
cannam@175:                 break;
cannam@175: 
cannam@175:             case Median:
cannam@180:                 if (continuous) result = summary.median_c;
cannam@180:                 else result = summary.median;
cannam@175:                 break;
cannam@175: 
cannam@175:             case Mode:
cannam@180:                 if (continuous) result = summary.mode_c;
cannam@180:                 else result = summary.mode;
cannam@175:                 break;
cannam@175: 
cannam@175:             case Sum:
cannam@175:                 result = summary.sum;
cannam@175:                 break;
cannam@175: 
cannam@175:             case Variance:
cannam@180:                 if (continuous) result = summary.variance_c;
cannam@180:                 else result = summary.variance;
cannam@175:                 break;
cannam@175: 
cannam@175:             case StandardDeviation:
cannam@180:                 if (continuous) result = sqrtf(summary.variance_c);
cannam@180:                 else result = sqrtf(summary.variance);
cannam@175:                 break;
cannam@175: 
cannam@175:             case Count:
cannam@175:                 result = summary.count;
cannam@175:                 break;
cannam@180: 
cannam@180:             default:
cannam@180:                 break;
cannam@175:             }
cannam@177:             
cannam@177:             f.values.push_back(result);
cannam@175:         }
cannam@175: 
cannam@175:         fl.push_back(f);
cannam@175:     }
cannam@175:     return fl;
cannam@175: }
cannam@175: 
cannam@176: Plugin::FeatureSet
cannam@180: PluginSummarisingAdapter::Impl::getSummaryForAllOutputs(SummaryType type,
cannam@180:                                                         AveragingMethod avg)
cannam@176: {
cannam@183:     if (!m_reduced) reduce();
cannam@183: 
cannam@176:     FeatureSet fs;
cannam@176:     for (OutputSummarySegmentMap::const_iterator i = m_summaries.begin();
cannam@176:          i != m_summaries.end(); ++i) {
cannam@180:         fs[i->first] = getSummaryForOutput(i->first, type, avg);
cannam@176:     }
cannam@176:     return fs;
cannam@176: }
cannam@176: 
cannam@174: void
cannam@174: PluginSummarisingAdapter::Impl::accumulate(const FeatureSet &fs,
cannam@180:                                            RealTime timestamp, 
cannam@180:                                            bool final)
cannam@174: {
cannam@174:     for (FeatureSet::const_iterator i = fs.begin(); i != fs.end(); ++i) {
cannam@174:         for (FeatureList::const_iterator j = i->second.begin();
cannam@174:              j != i->second.end(); ++j) {
cannam@182:             if (j->hasTimestamp) {
cannam@182:                 accumulate(i->first, *j, j->timestamp, final);
cannam@182:             } else {
cannam@182:                 //!!! is this correct?
cannam@182:                 accumulate(i->first, *j, timestamp, final);
cannam@182:             }
cannam@174:         }
cannam@174:     }
cannam@174: }
cannam@174: 
cannam@174: void
cannam@174: PluginSummarisingAdapter::Impl::accumulate(int output,
cannam@174:                                            const Feature &f,
cannam@180:                                            RealTime timestamp,
cannam@180:                                            bool final)
cannam@174: {
cannam@180: //!!! to do: use timestamp to determine which segment we're on
cannam@180: 
cannam@174:     m_accumulators[output].count++;
cannam@180: 
cannam@182:     std::cerr << "output " << output << ": timestamp " << timestamp << ", prev timestamp " << m_prevTimestamps[output] << std::endl;
cannam@182: 
cannam@182:     //!!! also, this will not work if we are called repeatedly with
cannam@182:     //!!! the same timestamp -- no values will be registered until a
cannam@182:     //!!! new timestamp is seen -- we need a better test for "not
cannam@182:     //!!! first result" below
cannam@182: 
cannam@183:     if (m_prevDurations[output] == RealTime::zeroTime) {
cannam@180:         if (m_prevTimestamps.find(output) != m_prevTimestamps.end()) {
cannam@183:             m_prevDurations[output] = timestamp - m_prevTimestamps[output];
cannam@180:         }
cannam@180:     }
cannam@183:     if (m_prevDurations[output] != RealTime::zeroTime ||
cannam@180:         !m_accumulators[output].durations.empty()) {
cannam@180:         // ... i.e. if not first result.  We don't push a duration
cannam@180:         // when we process the first result; then the duration we push
cannam@180:         // each time is that for the result before the one we're
cannam@180:         // processing, and we push an extra one at the end.  This
cannam@180:         // permits handling the case where the feature itself doesn't
cannam@180:         // have a duration field, and we have to calculate it from the
cannam@180:         // time to the following feature.  The net effect is simply
cannam@180:         // that values[n] and durations[n] refer to the same result.
cannam@183:         m_accumulators[output].durations.push_back(m_prevDurations[output]);
cannam@180:     }
cannam@180: 
cannam@180:     m_prevTimestamps[output] = timestamp;
cannam@180: 
cannam@174:     for (int i = 0; i < int(f.values.size()); ++i) {
cannam@174:         m_accumulators[output].values[i].push_back(f.values[i]);
cannam@174:     }
cannam@180: 
cannam@180:     if (final) {
cannam@180:         RealTime finalDuration;
cannam@180:         if (f.hasDuration) finalDuration = f.duration;
cannam@180:         m_accumulators[output].durations.push_back(finalDuration);
cannam@180:     }
cannam@180: 
cannam@183:     if (f.hasDuration) m_prevDurations[output] = f.duration;
cannam@183:     else m_prevDurations[output] = RealTime::zeroTime;
cannam@174: }
cannam@174: 
cannam@181: struct ValueDurationFloatPair
cannam@181: {
cannam@181:     float value;
cannam@181:     float duration;
cannam@181: 
cannam@181:     ValueDurationFloatPair() : value(0), duration(0) { }
cannam@181:     ValueDurationFloatPair(float v, float d) : value(v), duration(d) { }
cannam@181:     ValueDurationFloatPair &operator=(const ValueDurationFloatPair &p) {
cannam@181:         value = p.value;
cannam@181:         duration = p.duration;
cannam@181:         return *this;
cannam@181:     }
cannam@181:     bool operator<(const ValueDurationFloatPair &p) const {
cannam@181:         return value < p.value;
cannam@181:     }
cannam@181: };
cannam@181: 
cannam@181: static double toSec(const RealTime &r)
cannam@181: {
cannam@181:     return r.sec + double(r.nsec) / 1000000000.0;
cannam@181: }
cannam@181: 
cannam@174: void
cannam@174: PluginSummarisingAdapter::Impl::reduce()
cannam@174: {
cannam@174:     RealTime segmentStart = RealTime::zeroTime; //!!!
cannam@174: 
cannam@174:     for (OutputAccumulatorMap::iterator i = m_accumulators.begin();
cannam@174:          i != m_accumulators.end(); ++i) {
cannam@174: 
cannam@174:         int output = i->first;
cannam@174:         OutputAccumulator &accumulator = i->second;
cannam@174: 
cannam@182:         double totalDuration = 0.0;
cannam@181:         for (int k = 0; k < accumulator.durations.size(); ++k) {
cannam@181:             totalDuration += toSec(accumulator.durations[k]);
cannam@180:         }
cannam@180: 
cannam@174:         for (BinValueMap::iterator j = accumulator.values.begin();
cannam@174:              j != accumulator.values.end(); ++j) {
cannam@174: 
cannam@180:             // work on all values over time for a single bin
cannam@180: 
cannam@174:             int bin = j->first;
cannam@181:             const ValueList &values = j->second;
cannam@180:             const DurationList &durations = accumulator.durations;
cannam@174: 
cannam@174:             OutputBinSummary summary;
cannam@180: 
cannam@180:             summary.count = accumulator.count;
cannam@180: 
cannam@174:             summary.minimum = 0.f;
cannam@174:             summary.maximum = 0.f;
cannam@180: 
cannam@174:             summary.median = 0.f;
cannam@174:             summary.mode = 0.f;
cannam@174:             summary.sum = 0.f;
cannam@174:             summary.variance = 0.f;
cannam@180: 
cannam@180:             summary.median_c = 0.f;
cannam@180:             summary.mode_c = 0.f;
cannam@180:             summary.mean_c = 0.f;
cannam@180:             summary.variance_c = 0.f;
cannam@180: 
cannam@174:             if (summary.count == 0 || values.empty()) continue;
cannam@174: 
cannam@174:             int sz = values.size();
cannam@174: 
cannam@180:             if (sz != durations.size()) {
cannam@180:                 std::cerr << "WARNING: sz " << sz << " != durations.size() "
cannam@180:                           << durations.size() << std::endl;
cannam@181: //                while (durations.size() < sz) {
cannam@181: //                    durations.push_back(RealTime::zeroTime);
cannam@181: //                }
cannam@181: //!!! then what?
cannam@180:             }
cannam@180: 
cannam@181:             std::vector<ValueDurationFloatPair> valvec;
cannam@181: 
cannam@181:             for (int k = 0; k < sz; ++k) {
cannam@181:                 valvec.push_back(ValueDurationFloatPair(values[k],
cannam@181:                                                         toSec(durations[k])));
cannam@181:             }
cannam@181: 
cannam@181:             std::sort(valvec.begin(), valvec.end());
cannam@181: 
cannam@181:             summary.minimum = valvec[0].value;
cannam@181:             summary.maximum = valvec[sz-1].value;
cannam@174: 
cannam@174:             if (sz % 2 == 1) {
cannam@181:                 summary.median = valvec[sz/2].value;
cannam@174:             } else {
cannam@181:                 summary.median = (valvec[sz/2].value + valvec[sz/2 + 1].value) / 2;
cannam@174:             }
cannam@181:             
cannam@181:             double duracc = 0.0;
cannam@181:             summary.median_c = valvec[sz-1].value;
cannam@174: 
cannam@181:             for (int k = 0; k < sz; ++k) {
cannam@181:                 duracc += valvec[k].duration;
cannam@181:                 if (duracc > totalDuration/2) {
cannam@181:                     summary.median_c = valvec[k].value;
cannam@181:                     break;
cannam@181:                 }
cannam@181:             }
cannam@181:                 
cannam@174:             std::map<float, int> distribution;
cannam@174: 
cannam@174:             for (int k = 0; k < sz; ++k) {
cannam@174:                 summary.sum += values[k];
cannam@180:                 distribution[values[k]] += 1;
cannam@174:             }
cannam@174: 
cannam@174:             int md = 0;
cannam@174: 
cannam@174:             for (std::map<float, int>::iterator di = distribution.begin();
cannam@174:                  di != distribution.end(); ++di) {
cannam@174:                 if (di->second > md) {
cannam@174:                     md = di->second;
cannam@174:                     summary.mode = di->first;
cannam@174:                 }
cannam@174:             }
cannam@174: 
cannam@174:             distribution.clear();
cannam@174: 
cannam@180:             //!!! we want to omit this bit if the features all have
cannam@180:             //!!! equal duration (and set mode_c equal to mode instead)
cannam@180:             
cannam@181:             std::map<float, double> distribution_c;
cannam@180: 
cannam@180:             for (int k = 0; k < sz; ++k) {
cannam@181:                 distribution_c[values[k]] += toSec(durations[k]);
cannam@180:             }
cannam@180: 
cannam@181:             double mrd = 0.0;
cannam@180: 
cannam@181:             for (std::map<float, double>::iterator di = distribution_c.begin();
cannam@180:                  di != distribution_c.end(); ++di) {
cannam@180:                 if (di->second > mrd) {
cannam@180:                     mrd = di->second;
cannam@180:                     summary.mode_c = di->first;
cannam@180:                 }
cannam@180:             }
cannam@180: 
cannam@180:             distribution_c.clear();
cannam@180: 
cannam@181:             if (totalDuration > 0.0) {
cannam@181: 
cannam@181:                 double sum_c = 0.0;
cannam@181: 
cannam@181:                 for (int k = 0; k < sz; ++k) {
cannam@181:                     double value = values[k] * toSec(durations[k]);
cannam@181:                     sum_c += value;
cannam@181:                 }
cannam@182: 
cannam@182:                 std::cerr << "mean_c = " << sum_c << " / " << totalDuration << " = "
cannam@182:                           << sum_c / totalDuration << std::endl;
cannam@181:                 
cannam@181:                 summary.mean_c = sum_c / totalDuration;
cannam@181: 
cannam@181:                 for (int k = 0; k < sz; ++k) {
cannam@181:                     double value = values[k] * toSec(durations[k]);
cannam@181:                     summary.variance_c +=
cannam@181:                         (value - summary.mean_c) * (value - summary.mean_c);
cannam@181:                 }
cannam@181: 
cannam@181:                 summary.variance_c /= summary.count;
cannam@181:             }
cannam@181: 
cannam@181:             //!!! still to handle: median_c
cannam@180: 
cannam@174:             float mean = summary.sum / summary.count;
cannam@174: 
cannam@182:             std::cerr << "mean = " << summary.sum << " / " << summary.count << " = "
cannam@182:                       << summary.sum / summary.count << std::endl;
cannam@182: 
cannam@174:             for (int k = 0; k < sz; ++k) {
cannam@174:                 summary.variance += (values[k] - mean) * (values[k] - mean);
cannam@174:             }
cannam@174:             summary.variance /= summary.count;
cannam@174: 
cannam@174:             m_summaries[output][segmentStart][bin] = summary;
cannam@174:         }
cannam@174:     }
cannam@175: 
cannam@175:     m_accumulators.clear();
cannam@183:     m_reduced = true;
cannam@174: }
cannam@174: 
cannam@174: 
cannam@174: }
cannam@174: 
cannam@174: }
cannam@174: