Vamp Plugin Tester

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*

    Vamp Plugin Tester

    Chris Cannam, cannam@all-day-breakfast.com

    Centre for Digital Music, Queen Mary, University of London.

    Copyright 2009 QMUL.

    This program loads a Vamp plugin and tests its susceptibility to a

    number of common pitfalls, including handling of extremes of input

    data.  If you can think of any additional useful tests that are

    easily added, please send them to me.

    Permission is hereby granted, free of charge, to any person

    obtaining a copy of this software and associated documentation

    files (the "Software"), to deal in the Software without

    restriction, including without limitation the rights to use, copy,

    modify, merge, publish, distribute, sublicense, and/or sell copies

    of the Software, and to permit persons to whom the Software is

    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be

    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR

    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF

    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    Except as contained in this notice, the names of the Centre for

    Digital Music; Queen Mary, University of London; and Chris Cannam

    shall not be used in advertising or otherwise to promote the sale,

    use or other dealings in this Software without prior written

    authorization.

*/

#include <vamp-hostsdk/PluginHostAdapter.h>

#include <vamp-hostsdk/PluginInputDomainAdapter.h>

#include <vamp-hostsdk/PluginLoader.h>

#include <iostream>

#include <cstring>

#include <cstdlib>

#include <cmath>

#include <set>

#include "Tester.h"

using Vamp::Plugin;

using Vamp::PluginHostAdapter;

using Vamp::RealTime;

using Vamp::HostExt::PluginLoader;

using Vamp::HostExt::PluginWrapper;

using Vamp::HostExt::PluginInputDomainAdapter;

using namespace std;

Tester::Tester(std::string key, Test::Options options, std::string singleTestId) :

    m_key(key),

    m_options(options),

    m_singleTest(singleTestId)

{

}

Tester::~Tester()

{

}

Tester::NameIndex &

Tester::nameIndex()

{

    static NameIndex ix;

    return ix;

}

Tester::Registry &

Tester::registry()

{

    static Registry r;

    return r;

}

void

Tester::listTests()

{

    cout << endl;

    cout << "Total tests: " << nameIndex().size() << "\n" << endl;

    cout << "ID | Name" << endl;

    cout << "---+-----" << endl;

    for (NameIndex::const_iterator i = nameIndex().begin();

         i != nameIndex().end(); ++i) {

        cout << i->first << " | " << i->second << endl;

    }

    cout << endl;

}

bool

Tester::test(int &notes, int &warnings, int &errors)

{

    /*

      Things I would like to see tested:

      * Identifiers for parameters, outputs, or plugin itself contain

        illegal characters - DONE

      * Any of the plugin's name, maker etc fields are empty - DONE

      * Default value of a parameter is not quantized as specified - DONE

      * Parameter minValue >= maxValue, or defaultValue < minValue

        or > maxValue - DONE

      * Plugin fails when given zero-length or very short input - DONE

      * Plugin fails when given "all digital zeros" input - DONE

      * Plugin fails when given input that exceeds +/-1 - DONE

      * Plugin fails when given "normal" random input (just in case!) - DONE

      * Plugin returns different results if another instance is

        constructed and run "interleaved" with it (from same thread) - DONE

      * Plugin's returned timestamps do not change as expected when

        run with a different base timestamp for input (though there

        could be legitimate reasons for this) - DONE

      * Plugin produces different results on second run, after reset

        called - DONE

      * Initial value of a parameter on plugin construction differs

        from its default value (i.e. plugin produces different

        results depending on whether parameter is set explicitly by

        host to default value or not) - DONE

      * If a plugin reports any programs, selecting default program

        explicitly changes results (as for default parameters) - DONE

      * Output feature does not hasTimestamp when output type is

        VariableSampleRate - DONE

      * Output feature hasTimestamp or hasDuration when output type is

        OneSamplePerStep (warning only, this is not an error) - DONE

      * Plugin returns features whose output numbers do not have

        a corresponding record in output descriptor list - DONE

      * Plugin fails to return any features on some output (warning

        only) - DONE

      * Constructor takes a long time to run.  A fuzzy concept, but

        suggests that some work should have been deferred to

        initialise().  Warning only - DONE

      * Plugin fails gracelessly when constructed with "weird" sample

        rate or initialised with "wrong" step size, block size, or

        number of channels

      Well, that's quite a lot of tests already.  What else?

    */

    bool good = true;

    try {

        if (m_options & Test::SingleTest) {

            if (registry().find(m_singleTest) != registry().end()) {

                good = performTest(m_singleTest, notes, warnings, errors);

            } else {

                std::cout << " ** ERROR: Unknown single-test id \""

                          << m_singleTest << "\"" << std::endl;

                good = false;

            }

        } else {

            for (Registry::const_iterator i = registry().begin();

                 i != registry().end(); ++i) {

                bool thisGood = performTest(i->first, notes, warnings, errors);

                if (!thisGood) good = false;

            }

        }

    } catch (Test::FailedToLoadPlugin) {

        std::cout << " ** ERROR: Failed to load plugin (key = \"" << m_key

                  << "\")" << std::endl;

        std::cout << " ** NOTE: Vamp plugin path is: " << std::endl;

        std::vector<std::string> pp = PluginHostAdapter::getPluginPath();

        for (size_t i = 0; i < pp.size(); ++i) {

            std::cout << "            " << pp[i] << std::endl;

        }

        good = false;

    }

    return good;

}

bool

Tester::performTest(std::string id, int &notes, int &warnings, int &errors)

{

    std::cout << " -- Performing test: "

              << id

              << " "

              << nameIndex()[id]

              << std::endl;

    Test *test = registry()[id]->makeTest();

    Test::Results results = test->test(m_key, m_options);

    delete test;

    set<string> printed;

    bool good = true;

    for (int j = 0; j < (int)results.size(); ++j) {

        string message = results[j].message();

        if (printed.find(message) != printed.end()) continue;

        printed.insert(message);

        switch (results[j].code()) {

        case Test::Result::Success:

            break;

        case Test::Result::Note:

            std::cout << " ** NOTE: " << results[j].message() << std::endl;

            ++notes;

            break;

        case Test::Result::Warning:

            std::cout << " ** WARNING: " << results[j].message() << std::endl;

            ++warnings;

            break;

        case Test::Result::Error:

            std::cout << " ** ERROR: " << results[j].message() << std::endl;

            ++errors;

            good = false;

            break;

        }

    }

    return good;

}