cannam@0: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
cannam@0: 
cannam@0: /*
cannam@0:     Vamp Plugin Fuzz Tester
cannam@0:     Chris Cannam, cannam@all-day-breakfast.com
cannam@0:     Centre for Digital Music, Queen Mary, University of London.
Chris@42:     Copyright 2009-2014 QMUL.
cannam@0: 
cannam@0:     This program loads a Vamp plugin and tests its susceptibility to a
cannam@0:     number of common pitfalls, including handling of extremes of input
cannam@0:     data.  If you can think of any additional useful tests that are
cannam@0:     easily added, please send them to me.
cannam@0:   
cannam@0:     Permission is hereby granted, free of charge, to any person
cannam@0:     obtaining a copy of this software and associated documentation
cannam@0:     files (the "Software"), to deal in the Software without
cannam@0:     restriction, including without limitation the rights to use, copy,
cannam@0:     modify, merge, publish, distribute, sublicense, and/or sell copies
cannam@0:     of the Software, and to permit persons to whom the Software is
cannam@0:     furnished to do so, subject to the following conditions:
cannam@0: 
cannam@0:     The above copyright notice and this permission notice shall be
cannam@0:     included in all copies or substantial portions of the Software.
cannam@0: 
cannam@0:     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
cannam@0:     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
cannam@0:     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
cannam@0:     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
cannam@0:     ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
cannam@0:     CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
cannam@0:     WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
cannam@0: 
cannam@0:     Except as contained in this notice, the names of the Centre for
cannam@0:     Digital Music; Queen Mary, University of London; and Chris Cannam
cannam@0:     shall not be used in advertising or otherwise to promote the sale,
cannam@0:     use or other dealings in this Software without prior written
cannam@0:     authorization.
cannam@0: */
cannam@0: 
cannam@0: #include "Test.h"
cannam@0: 
cannam@0: #include <vamp-hostsdk/PluginLoader.h>
cannam@0: 
cannam@0: using namespace Vamp;
cannam@0: using namespace Vamp::HostExt;
cannam@0: 
cannam@20: #include <math.h>
cannam@1: 
cannam@28: #ifdef __SUNPRO_CC
cannam@28: #include <ieeefp.h>
cannam@28: #define isinf(x) (!finite(x))
cannam@28: #endif
cannam@28: 
cannam@0: Test::Test() { }
cannam@0: Test::~Test() { }
cannam@0: 
Chris@52: using std::cerr;
Chris@52: using std::cout;
Chris@52: using std::endl;
Chris@52: using std::string;
Chris@52: 
cannam@0: Plugin *
Chris@52: Test::load(string key, float rate)
cannam@0: {
cannam@23:     Plugin *p = PluginLoader::getInstance()->loadPlugin
cannam@0:         (key, rate, PluginLoader::ADAPT_ALL);
cannam@23:     if (!p) throw FailedToLoadPlugin();
cannam@23:     return p;
cannam@0: }
cannam@0: 
cannam@1: float **
cannam@1: Test::createBlock(size_t channels, size_t blocksize)
cannam@1: {
cannam@1:     float **b = new float *[channels];
cannam@1:     for (size_t c = 0; c < channels; ++c) {
cannam@1:         b[c] = new float[blocksize];
cannam@1:     }
cannam@1:     return b;
cannam@1: }
cannam@1: 
cannam@1: void
cannam@1: Test::destroyBlock(float **blocks, size_t channels)
cannam@1: {
cannam@1:     for (size_t c = 0; c < channels; ++c) {
cannam@1:         delete[] blocks[c];
cannam@1:     }
cannam@1:     delete[] blocks;
cannam@1: }
cannam@1: 
cannam@3: float **
cannam@3: Test::createTestAudio(size_t channels, size_t blocksize, size_t blocks)
cannam@3: {
cannam@3:     float **b = new float *[channels];
cannam@3:     for (size_t c = 0; c < channels; ++c) {
cannam@3:         b[c] = new float[blocksize * blocks];
cannam@3:         for (int i = 0; i < int(blocksize * blocks); ++i) {
cannam@3:             b[c][i] = sinf(float(i) / 10.f);
cannam@3:             if (i == 5005 || i == 20002) {
cannam@3:                 b[c][i-2] = 0;
cannam@3:                 b[c][i-1] = -1;
cannam@3:                 b[c][i] = 1;
cannam@3:             }
cannam@3:         }
cannam@3:     }
cannam@3:     return b;
cannam@3: }
cannam@3: 
cannam@3: void
cannam@3: Test::destroyTestAudio(float **b, size_t channels)
cannam@3: {
cannam@3:     for (size_t c = 0; c < channels; ++c) {
cannam@3:         delete[] b[c];
cannam@3:     }
cannam@3:     delete[] b;
cannam@3: }
cannam@3: 
cannam@1: bool
cannam@1: Test::initDefaults(Plugin *p, size_t &channels, size_t &step, size_t &block,
cannam@1:                    Results &r)
cannam@1: {
cannam@1:     channels = p->getMinChannelCount();
cannam@1:     block = p->getPreferredBlockSize();
cannam@1:     step = p->getPreferredStepSize();
cannam@1:     if (block == 0) block = 1024;
cannam@1:     if (step == 0) {
cannam@1:         if (p->getInputDomain() == Plugin::FrequencyDomain) step = block/2;
cannam@1:         else step = block;
cannam@1:     }
cannam@1:     if (!p->initialise(channels, step, block)) {
cannam@1:         r.push_back(error("initialisation with default values failed"));
cannam@1:         return false;
cannam@1:     }
cannam@1:     return true;
cannam@1: }
cannam@1: 
cannam@3: bool
cannam@3: Test::initAdapted(Plugin *p, size_t &channels, size_t step, size_t block,
cannam@3:                   Results &r)
cannam@3: {
cannam@3:     channels = p->getMinChannelCount();
cannam@3:     if (!p->initialise(channels, step, block)) {
cannam@3:         r.push_back(error("initialisation failed"));
cannam@3:         return false;
cannam@3:     }
cannam@3:     return true;
cannam@3: }
cannam@3: 
cannam@0: void
cannam@0: Test::appendFeatures(Plugin::FeatureSet &a, const Plugin::FeatureSet &b)
cannam@0: {
cannam@0:     for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {
cannam@0:         int output = i->first;
cannam@0:         const Plugin::FeatureList &fl = i->second;
cannam@0:         Plugin::FeatureList &target = a[output];
cannam@0:         for (Plugin::FeatureList::const_iterator j = fl.begin(); j != fl.end(); ++j) {
cannam@0:             target.push_back(*j);
cannam@0:         }
cannam@0:     }
cannam@0: }
cannam@0: 
cannam@0: bool
cannam@1: Test::allFeaturesValid(const Plugin::FeatureSet &b)
cannam@1: {
cannam@1:     for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {
cannam@1:         for (int j = 0; j < (int)i->second.size(); ++j) {
cannam@1:             if (i->second[j].values.empty()) continue;
cannam@1:             for (int k = 0; k < (int)i->second[j].values.size(); ++k) {
cannam@1:                 if (isnan(i->second[j].values[k]) ||
cannam@1:                     isinf(i->second[j].values[k])) {
cannam@1:                     return false;
cannam@1:                 }
cannam@1:             }
cannam@1:         }
cannam@1:     }
cannam@1:     return true;
cannam@1: }
cannam@1: 
Chris@53: bool
Chris@53: Test::containsTimestamps(const Plugin::FeatureSet &b)
Chris@53: {
Chris@53:     for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {
Chris@53:         for (int j = 0; j < (int)i->second.size(); ++j) {
Chris@53:             if (i->second[j].values.empty()) continue;
Chris@53:             for (int k = 0; k < (int)i->second[j].values.size(); ++k) {
Chris@53:                 if (i->second[j].hasTimestamp) {
Chris@53:                     return true;
Chris@53:                 }
Chris@53:             }
Chris@53:         }
Chris@53:     }
Chris@53:     return false;
Chris@53: }
Chris@53: 
cannam@3: void
Chris@61: Test::dumpFeature(const Plugin::Feature &f, bool showValues,
Chris@61:                   const Plugin::Feature *other)
Chris@52: {
Chris@52:     cout << "    Timestamp: " << (!f.hasTimestamp ? "(none)" : f.timestamp.toText()) << endl;
Chris@52:     cout << "    Duration: " << (!f.hasDuration ? "(none)" : f.duration.toText()) << endl;
Chris@52:     cout << "    Label: " << (f.label == "" ? "(none)" : f.label) << endl;
Chris@52:     if (showValues) {
Chris@52:         cout << "    Values (" << f.values.size() << "): " << (f.values.empty() ? "(none)" : "");
Chris@61:         int n = f.values.size();
Chris@61:         if (!other) {
Chris@61:             for (int j = 0; j < n; ++j) {
Chris@61:                 cout << f.values[j] << " ";
Chris@61:             }
Chris@61:         } else {
Chris@61:             int samecount = 0;
Chris@61:             int diffcount = 0;
Chris@61:             for (int j = 0; j <= n; ++j) {
Chris@61:                 if (j < n && f.values[j] == other->values[j]) {
Chris@61:                     ++samecount;
Chris@61:                 } else {
Chris@61:                     if (samecount > 0) {
Chris@61:                         cout << "(" << samecount << " identical) ";
Chris@61:                     }
Chris@61:                     samecount = 0;
Chris@61:                     if (j < n) {
Chris@61:                         ++diffcount;
Chris@61:                         if (diffcount > 20 && j + 10 < n) {
Chris@61:                             cout << "(remaining " << n - j << " values elided)";
Chris@61:                             break;
Chris@61:                         } else {
Chris@61:                             cout << f.values[j] << " [diff "
Chris@61:                                  << f.values[j] - other->values[j] << "] ";
Chris@61:                         }
Chris@61:                     }
Chris@61:                 }
Chris@61:             }
Chris@52:         }
Chris@52:         cout << endl;
Chris@52:     } else {
Chris@52:         cout << "    Values (" << f.values.size() << "): (elided)" << endl;
Chris@52:     }
Chris@52: }    
Chris@52: 
Chris@52: void
Chris@52: Test::dump(const Plugin::FeatureSet &fs, bool showValues)
cannam@3: {
cannam@3:     for (Plugin::FeatureSet::const_iterator fsi = fs.begin();
cannam@3:          fsi != fs.end(); ++fsi) {
cannam@3:         int output = fsi->first;
Chris@52:         cout << "Output " << output << ":" << endl;
cannam@3:         const Plugin::FeatureList &fl = fsi->second;
cannam@3:         for (int i = 0; i < (int)fl.size(); ++i) {
Chris@52:             cout << "  Feature " << i << ":" << endl;
cannam@3:             const Plugin::Feature &f = fl[i];
Chris@52:             dumpFeature(f, showValues);
cannam@3:         }
cannam@3:     }
cannam@3: }
cannam@3: 
cannam@3: void
Chris@52: Test::dumpTwo(const Result &r,
Chris@52:               const Plugin::FeatureSet &a,
Chris@52:               const Plugin::FeatureSet &b)
cannam@3: {
cannam@8:     std::cout << r.message() << std::endl;
cannam@8:     std::cout << "\nFirst result set:" << std::endl;
Chris@52:     dump(a, false);
cannam@8:     std::cout << "\nSecond result set:" << std::endl;
Chris@52:     dump(b, false);
cannam@8:     std::cout << std::endl;
cannam@3: }
cannam@3: 
Chris@52: void
Chris@52: Test::dumpDiff(const Result &r,
Chris@52:                const Plugin::FeatureSet &a,
Chris@52:                const Plugin::FeatureSet &b)
Chris@52: {
Chris@52:     cout << r.message() << endl;
Chris@52:     cout << "\nDifferences follow:" << endl;
Chris@52:     if (a.size() != b.size()) {
Chris@52:         cout << "*** First result set has features on " << a.size() 
Chris@52:                   << " output(s), second has features on " << b.size()
Chris@52:                   << endl;
Chris@52:         return;
Chris@52:     }
Chris@52:     Plugin::FeatureSet::const_iterator ai = a.begin();
Chris@52:     Plugin::FeatureSet::const_iterator bi = b.begin();
Chris@52:     while (ai != a.end()) {
Chris@52:         if (ai->first != bi->first) {
Chris@52:             cout << "\n*** Output number mismatch: first result set says "
Chris@52:                       << ai->first << " where second says " << bi->first
Chris@52:                       << endl;
Chris@52:         } else {
Chris@52:             cout << "\nOutput " << ai->first << ":" << endl;
Chris@52:             if (ai->second.size() != bi->second.size()) {
Chris@52:                 cout << "*** First result set has " << ai->second.size()
Chris@52:                           << " feature(s) on this output, second has "
Chris@52:                           << bi->second.size() << endl;
Chris@52:             } else {
Chris@52:                 int fno = 0;
Chris@52:                 int diffcount = 0;
Chris@52:                 Plugin::FeatureList::const_iterator afi = ai->second.begin();
Chris@52:                 Plugin::FeatureList::const_iterator bfi = bi->second.begin();
Chris@52:                 while (afi != ai->second.end()) {
Chris@52:                     if (!(*afi == *bfi)) {
Chris@52:                         if (diffcount == 0) {
Chris@61:                             bool differInValues =
Chris@61:                                 (afi->values.size() == bfi->values.size() &&
Chris@61:                                  afi->values != bfi->values);
Chris@52:                             if (afi->hasTimestamp != bfi->hasTimestamp) {
Chris@52:                                 cout << "*** Feature " << fno << " differs in presence of timestamp (" << afi->hasTimestamp << " vs " << bfi->hasTimestamp << ")" << endl;
Chris@52:                             }
Chris@52:                             if (afi->hasTimestamp && (afi->timestamp != bfi->timestamp)) {
Chris@52:                                 cout << "*** Feature " << fno << " differs in timestamp (" << afi->timestamp << " vs " << bfi->timestamp << " )" << endl;
Chris@52:                             }
Chris@52:                             if (afi->hasDuration != bfi->hasDuration) {
Chris@52:                                 cout << "*** Feature " << fno << " differs in presence of duration (" << afi->hasDuration << " vs " << bfi->hasDuration << ")" << endl;
Chris@52:                             }
Chris@52:                             if (afi->hasDuration && (afi->duration != bfi->duration)) {
Chris@52:                                 cout << "*** Feature " << fno << " differs in duration (" << afi->duration << " vs " << bfi->duration << " )" << endl;
Chris@52:                             }
Chris@52:                             if (afi->label != bfi->label) {
Chris@52:                                 cout << "*** Feature " << fno << " differs in label" << endl;
Chris@52:                             }
Chris@61:                             if (afi->values.size() != bfi->values.size()) {
Chris@61:                                 cout << "*** Feature " << fno << " differs in number of values (" << afi->values.size() << " vs " << bfi->values.size() << ")" << endl;
Chris@61:                             }
Chris@52:                             if (differInValues) {
Chris@52:                                 cout << "*** Feature " << fno << " differs in values" << endl;
Chris@52:                             }
Chris@52:                             cout << "  First output:" << endl;
Chris@52:                             dumpFeature(*afi, differInValues);
Chris@52:                             cout << "  Second output:" << endl;
Chris@61:                             dumpFeature(*bfi, differInValues, &(*afi));
Chris@52:                         }
Chris@52:                         ++diffcount;
Chris@52:                     }
Chris@52:                     ++fno;
Chris@52:                     ++afi;
Chris@52:                     ++bfi;
Chris@52:                 }
Chris@52:                 if (diffcount > 1) {
Chris@52:                     cout << diffcount-1 << " subsequent differing feature(s) elided" << endl;
Chris@52:                 }
Chris@52:             }
Chris@52:         }                
Chris@52:         ++ai;
Chris@52:         ++bi;
Chris@52:     }
Chris@52:     cout << endl;
Chris@52: }
Chris@52: 
cannam@1: bool
cannam@0: operator==(const Plugin::FeatureSet &a, const Plugin::FeatureSet &b)
cannam@0: {
cannam@0:     if (a.size() != b.size()) return false;
cannam@0:     for (Plugin::FeatureSet::const_iterator ai = a.begin();
cannam@0:          ai != a.end(); ++ai) {
cannam@0:         int output = ai->first;
cannam@0:         Plugin::FeatureSet::const_iterator bi = b.find(output);
cannam@0:         if (bi == b.end()) return false;
cannam@0:         if (!(ai->second == bi->second)) return false;
cannam@0:     }
cannam@0:     return true;
cannam@0: }
cannam@0: 
cannam@0: bool
cannam@0: operator==(const Plugin::FeatureList &a, const Plugin::FeatureList &b)
cannam@0: {
cannam@0:     if (a.size() != b.size()) return false;
cannam@0:     for (int i = 0; i < (int)a.size(); ++i) {
cannam@0:         if (!(a[i] == b[i])) return false;
cannam@0:     }
cannam@0:     return true;
cannam@0: }
cannam@0: 
cannam@0: bool
cannam@0: operator==(const Plugin::Feature &a, const Plugin::Feature &b)
cannam@0: {
cannam@0:     if (a.hasTimestamp != b.hasTimestamp) return false;
cannam@0:     if (a.hasTimestamp && (a.timestamp != b.timestamp)) return false;
cannam@0:     if (a.hasDuration != b.hasDuration) return false;
cannam@0:     if (a.hasDuration && (a.duration != b.duration)) return false;
cannam@0:     if (a.values != b.values) return false;
cannam@0:     if (a.label != b.label) return false;
cannam@0:     return true;
cannam@0: }
cannam@0: