cannam@0: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
cannam@0: 
cannam@0: /*
cannam@0:     Vamp Plugin Tester
cannam@0:     Chris Cannam, cannam@all-day-breakfast.com
cannam@0:     Centre for Digital Music, Queen Mary, University of London.
Chris@42:     Copyright 2009-2014 QMUL.
cannam@0: 
cannam@0:     This program loads a Vamp plugin and tests its susceptibility to a
cannam@0:     number of common pitfalls, including handling of extremes of input
cannam@0:     data.  If you can think of any additional useful tests that are
cannam@0:     easily added, please send them to me.
cannam@0:   
cannam@0:     Permission is hereby granted, free of charge, to any person
cannam@0:     obtaining a copy of this software and associated documentation
cannam@0:     files (the "Software"), to deal in the Software without
cannam@0:     restriction, including without limitation the rights to use, copy,
cannam@0:     modify, merge, publish, distribute, sublicense, and/or sell copies
cannam@0:     of the Software, and to permit persons to whom the Software is
cannam@0:     furnished to do so, subject to the following conditions:
cannam@0: 
cannam@0:     The above copyright notice and this permission notice shall be
cannam@0:     included in all copies or substantial portions of the Software.
cannam@0: 
cannam@0:     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
cannam@0:     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
cannam@0:     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
cannam@0:     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
cannam@0:     ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
cannam@0:     CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
cannam@0:     WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
cannam@0: 
cannam@0:     Except as contained in this notice, the names of the Centre for
cannam@0:     Digital Music; Queen Mary, University of London; and Chris Cannam
cannam@0:     shall not be used in advertising or otherwise to promote the sale,
cannam@0:     use or other dealings in this Software without prior written
cannam@0:     authorization.
cannam@0: */
cannam@0: 
cannam@0: #include <vamp-hostsdk/PluginHostAdapter.h>
cannam@0: #include <vamp-hostsdk/PluginInputDomainAdapter.h>
cannam@0: #include <vamp-hostsdk/PluginLoader.h>
cannam@0: 
cannam@0: #include <iostream>
cannam@0: 
cannam@0: #include <cstring>
cannam@0: #include <cstdlib>
cannam@0: #include <cmath>
cannam@4: #include <set>
cannam@0: 
cannam@0: #include "Tester.h"
cannam@0: 
cannam@0: using Vamp::Plugin;
cannam@0: using Vamp::PluginHostAdapter;
cannam@0: using Vamp::RealTime;
cannam@0: using Vamp::HostExt::PluginLoader;
cannam@0: using Vamp::HostExt::PluginWrapper;
cannam@0: using Vamp::HostExt::PluginInputDomainAdapter;
cannam@0: 
cannam@0: using namespace std;
cannam@0: 
Chris@39: Tester::Tester(std::string key, Test::Options options, std::string singleTestId) :
cannam@8:     m_key(key),
Chris@39:     m_options(options),
Chris@39:     m_singleTest(singleTestId)
cannam@0: {
cannam@0: }
cannam@0: 
cannam@0: Tester::~Tester()
cannam@0: {
cannam@0: }
cannam@0: 
Chris@39: Tester::NameIndex &
Chris@39: Tester::nameIndex()
Chris@39: {
Chris@39:     static NameIndex ix;
Chris@39:     return ix;
Chris@39: }
Chris@39: 
cannam@0: Tester::Registry &
cannam@0: Tester::registry()
cannam@0: {
cannam@0:     static Registry r;
cannam@0:     return r;
cannam@0: }
cannam@0: 
Chris@40: void
Chris@40: Tester::listTests()
Chris@40: {
Chris@40:     cout << endl;
Chris@40:     cout << "Total tests: " << nameIndex().size() << "\n" << endl;
Chris@40:     cout << "ID | Name" << endl;
Chris@40:     cout << "---+-----" << endl;
Chris@40:     for (NameIndex::const_iterator i = nameIndex().begin();
Chris@40:          i != nameIndex().end(); ++i) {
Chris@40:         cout << i->first << " | " << i->second << endl;
Chris@40:     }
Chris@40:     cout << endl;
Chris@40: }
Chris@40: 
cannam@0: bool
cannam@4: Tester::test(int &notes, int &warnings, int &errors)
cannam@0: {
cannam@0:     /*
cannam@0:       
cannam@0:       Things I would like to see tested:
cannam@0: 
cannam@0:       * Identifiers for parameters, outputs, or plugin itself contain
cannam@0:         illegal characters - DONE
cannam@0: 
cannam@0:       * Any of the plugin's name, maker etc fields are empty - DONE
cannam@0: 
cannam@1:       * Default value of a parameter is not quantized as specified - DONE
cannam@1: 
cannam@1:       * Parameter minValue >= maxValue, or defaultValue < minValue
cannam@1:         or > maxValue - DONE
cannam@1: 
cannam@1:       * Plugin fails when given zero-length or very short input - DONE
cannam@1: 
cannam@1:       * Plugin fails when given "all digital zeros" input - DONE
cannam@1: 
cannam@1:       * Plugin fails when given input that exceeds +/-1 - DONE
cannam@1: 
cannam@1:       * Plugin fails when given "normal" random input (just in case!) - DONE
cannam@1: 
cannam@0:       * Plugin returns different results if another instance is
cannam@3:         constructed and run "interleaved" with it (from same thread) - DONE
cannam@0:  
cannam@0:       * Plugin's returned timestamps do not change as expected when
cannam@0:         run with a different base timestamp for input (though there
cannam@5:         could be legitimate reasons for this) - DONE
cannam@0: 
cannam@0:       * Plugin produces different results on second run, after reset
cannam@3:         called - DONE
cannam@0: 
cannam@0:       * Initial value of a parameter on plugin construction differs
cannam@0:         from its default value (i.e. plugin produces different
cannam@0:         results depending on whether parameter is set explicitly by
cannam@6:         host to default value or not) - DONE
cannam@0:         
cannam@0:       * If a plugin reports any programs, selecting default program
cannam@6:         explicitly changes results (as for default parameters) - DONE
cannam@0: 
cannam@0:       * Output feature does not hasTimestamp when output type is
cannam@5:         VariableSampleRate - DONE
cannam@0: 
cannam@0:       * Output feature hasTimestamp or hasDuration when output type is
cannam@5:         OneSamplePerStep (warning only, this is not an error) - DONE
cannam@0: 
cannam@0:       * Plugin returns features whose output numbers do not have
cannam@5:         a corresponding record in output descriptor list - DONE
cannam@0: 
cannam@0:       * Plugin fails to return any features on some output (warning
cannam@5:         only) - DONE
cannam@0: 
cannam@0:       * Constructor takes a long time to run.  A fuzzy concept, but
cannam@0:         suggests that some work should have been deferred to
cannam@6:         initialise().  Warning only - DONE
cannam@6: 
cannam@6:       * Plugin fails gracelessly when constructed with "weird" sample
cannam@6:         rate or initialised with "wrong" step size, block size, or
cannam@6:         number of channels
cannam@0: 
cannam@0:       Well, that's quite a lot of tests already.  What else?
cannam@0: 
cannam@0:     */
cannam@0: 
cannam@0:     bool good = true;
cannam@0: 
cannam@0:     try {
cannam@0: 
Chris@39:         if (m_options & Test::SingleTest) {
cannam@4: 
Chris@39:             if (registry().find(m_singleTest) != registry().end()) {
Chris@39: 
Chris@39:                 good = performTest(m_singleTest, notes, warnings, errors);
Chris@39: 
Chris@39:             } else {
Chris@39: 
Chris@39:                 std::cout << " ** ERROR: Unknown single-test id \""
Chris@39:                           << m_singleTest << "\"" << std::endl;
Chris@39:                 good = false;
Chris@39:             }
Chris@39: 
Chris@39:         } else {
Chris@39:         
Chris@39:             for (Registry::const_iterator i = registry().begin();
Chris@39:                  i != registry().end(); ++i) {
Chris@39: 
Chris@39:                 bool thisGood = performTest(i->first, notes, warnings, errors);
Chris@39:                 if (!thisGood) good = false;
cannam@0:             }
cannam@0:         }
Chris@39: 
cannam@0:     } catch (Test::FailedToLoadPlugin) {
cannam@17:         std::cout << " ** ERROR: Failed to load plugin (key = \"" << m_key
cannam@0:                   << "\")" << std::endl;
cannam@25:         std::cout << " ** NOTE: Vamp plugin path is: " << std::endl;
cannam@25:         std::vector<std::string> pp = PluginHostAdapter::getPluginPath();
cannam@25:         for (size_t i = 0; i < pp.size(); ++i) {
cannam@25:             std::cout << "            " << pp[i] << std::endl;
cannam@25:         }
cannam@26:         good = false;
cannam@0:     }
cannam@0: 
cannam@0:     return good;
cannam@0: }
cannam@0: 
Chris@39: bool
Chris@39: Tester::performTest(std::string id, int &notes, int &warnings, int &errors)
Chris@39: {
Chris@39:     std::cout << " -- Performing test: "
Chris@39:               << id
Chris@39:               << " "
Chris@39:               << nameIndex()[id]
Chris@39:               << std::endl;
Chris@39:     
Chris@39:     Test *test = registry()[id]->makeTest();
Chris@39:     Test::Results results = test->test(m_key, m_options);
Chris@39:     delete test;
Chris@39: 
Chris@39:     set<string> printed;
Chris@39: 
Chris@39:     bool good = true;
Chris@39: 
Chris@39:     for (int j = 0; j < (int)results.size(); ++j) {
Chris@39:         string message = results[j].message();
Chris@39:         if (printed.find(message) != printed.end()) continue;
Chris@39:         printed.insert(message);
Chris@39:         switch (results[j].code()) {
Chris@39:         case Test::Result::Success:
Chris@39:             break;
Chris@39:         case Test::Result::Note:
Chris@39:             std::cout << " ** NOTE: " << results[j].message() << std::endl;
Chris@39:             ++notes;
Chris@39:             break;
Chris@39:         case Test::Result::Warning:
Chris@39:             std::cout << " ** WARNING: " << results[j].message() << std::endl;
Chris@39:             ++warnings;
Chris@39:             break;
Chris@39:         case Test::Result::Error:
Chris@39:             std::cout << " ** ERROR: " << results[j].message() << std::endl;
Chris@39:             ++errors;
Chris@39:             good = false;
Chris@39:             break;
Chris@39:         }
Chris@39:     }
Chris@39: 
Chris@39:     return good;
Chris@39: }
Chris@39: