Vamp Plugin Tester

OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o TestInputExtremes.o TestMultipleRuns.o TestOutputs.o

TestInputExtremes.o: TestInputExtremes.h Test.h Tester.h

TestMultipleRuns.o: TestMultipleRuns.h Test.h Tester.h

TestOutputs.o: TestOutputs.h Test.h Tester.h

TestInputExtremes.o: Test.h Tester.h

TestMultipleRuns.o: Test.h Tester.h

TestOutputs.o: Test.h Tester.h

float **

Test::createTestAudio(size_t channels, size_t blocksize, size_t blocks)

{

    float **b = new float *[channels];

    for (size_t c = 0; c < channels; ++c) {

        b[c] = new float[blocksize * blocks];

        for (int i = 0; i < int(blocksize * blocks); ++i) {

            b[c][i] = sinf(float(i) / 10.f);

            if (i == 5005 || i == 20002) {

                b[c][i-2] = 0;

                b[c][i-1] = -1;

                b[c][i] = 1;

            }

        }

    }

    return b;

}

void

Test::destroyTestAudio(float **b, size_t channels)

{

    for (size_t c = 0; c < channels; ++c) {

        delete[] b[c];

    }

    delete[] b;

}

bool

Test::initAdapted(Plugin *p, size_t &channels, size_t step, size_t block,

                  Results &r)

{

    channels = p->getMinChannelCount();

    if (!p->initialise(channels, step, block)) {

        r.push_back(error("initialisation failed"));

        return false;

    }

    return true;

}

void

Test::dump(const Plugin::FeatureSet &fs)

{

    for (Plugin::FeatureSet::const_iterator fsi = fs.begin();

         fsi != fs.end(); ++fsi) {

        int output = fsi->first;

        std::cerr << "Output " << output << ":" << std::endl;

        const Plugin::FeatureList &fl = fsi->second;

        for (int i = 0; i < (int)fl.size(); ++i) {

            std::cerr << "  Feature " << i << ":" << std::endl;

            const Plugin::Feature &f = fl[i];

            std::cerr << "    Timestamp: " << (f.hasTimestamp ? "(none)" : f.timestamp.toText()) << std::endl;

            std::cerr << "    Duration: " << (f.hasDuration ? "(none)" : f.duration.toText()) << std::endl;

            std::cerr << "    Label: " << (f.label == "" ? "(none)" : f.label) << std::endl;

            std::cerr << "    Value: " << (f.values.empty() ? "(none)" : "");

            for (int j = 0; j < (int)f.values.size(); ++j) {

                std::cerr << f.values[j] << " ";

            }

            std::cerr << std::endl;

        }

    }

}

void

Test::dump(const Result &r,

           const Plugin::FeatureSet &a,

           const Plugin::FeatureSet &b)

{

    std::cerr << r.message() << std::endl;

    std::cerr << "\nFirst result set:" << std::endl;

    dump(a);

    std::cerr << "\nSecond result set:" << std::endl;

    dump(b);

    std::cerr << std::endl;

}

        enum Code { Success, Note, Warning, Error };

        Code code() const { return m_code; }

        std::string message() const { return m_message; }

    static Result note(std::string m) { return Result(Result::Note, m); }

    float **createTestAudio(size_t channels, size_t blocksize, size_t blocks);

    void destroyTestAudio(float **audio, size_t channels);

    bool initAdapted(Vamp::Plugin *, size_t &channels,

                     size_t step, size_t block, Results &r);

    void dump(const Vamp::Plugin::FeatureSet &);

    void dump(const Result &r,

              const Vamp::Plugin::FeatureSet &,

              const Vamp::Plugin::FeatureSet &);

static const size_t _step = 1000;

    float **data = 0;

    size_t channels = 0;

    size_t count = 100;

        if (!initAdapted(p.get(), channels, _step, _step, r)) return r;

        if (!data) data = createTestAudio(channels, _step, count);

        for (size_t i = 0; i < count; ++i) {

            float *ptr[channels];

            size_t idx = i * _step;

            for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;

            Plugin::FeatureSet fs = p->process(ptr, timestamp);

    if (data) destroyTestAudio(data, channels);

    float **data = 0;

    size_t channels = 0;

    size_t count = 100;

        if (!initAdapted(p.get(), channels, _step, _step, r)) return r;

        if (!data) data = createTestAudio(channels, _step, count);

        for (size_t i = 0; i < count; ++i) {

            float *ptr[channels];

            size_t idx = i * _step;

            for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;

            Plugin::FeatureSet fs = p->process(ptr, timestamp);

    if (data) destroyTestAudio(data, channels);

        Result res = warning("Consecutive runs with the same instance (using reset) produce different results");

        dump(res, f[0], f[1]);

        r.push_back(res);

    float **data = 0;

    size_t channels = 0;

    size_t count = 100;

        if (!initAdapted(p[run], channels, _step, _step, r)) {

        if (!data) data = createTestAudio(channels, _step, count);

    for (size_t i = 0; i < count; ++i) {

        float *ptr[channels];

        size_t idx = i * _step;

        for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;

            Plugin::FeatureSet fs = p[run]->process(ptr, timestamp);

    if (data) destroyTestAudio(data, channels);

        r.push_back(warning("Simultaneous runs with separate instances produce different results"));

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*

    Vamp Plugin Tester

    Chris Cannam, cannam@all-day-breakfast.com

    Centre for Digital Music, Queen Mary, University of London.

    Copyright 2009 QMUL.

    This program loads a Vamp plugin and tests its susceptibility to a

    number of common pitfalls, including handling of extremes of input

    data.  If you can think of any additional useful tests that are

    easily added, please send them to me.

    Permission is hereby granted, free of charge, to any person

    obtaining a copy of this software and associated documentation

    files (the "Software"), to deal in the Software without

    restriction, including without limitation the rights to use, copy,

    modify, merge, publish, distribute, sublicense, and/or sell copies

    of the Software, and to permit persons to whom the Software is

    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be

    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR

    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF

    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    Except as contained in this notice, the names of the Centre for

    Digital Music; Queen Mary, University of London; and Chris Cannam

    shall not be used in advertising or otherwise to promote the sale,

    use or other dealings in this Software without prior written

    authorization.

*/

#include "TestOutputs.h"

#include <vamp-hostsdk/Plugin.h>

using namespace Vamp;

#include <set>

#include <memory>

using namespace std;

#include <cmath>

Tester::TestRegistrar<TestOutputNumbers>

TestOutputNumbers::m_registrar("Output number mismatching");

Tester::TestRegistrar<TestTimestamps>

TestTimestamps::m_registrar("Invalid or dubious timestamp usage");

static const size_t _step = 1000;

Test::Results

TestOutputNumbers::test(string key)

{

    int rate = 44100;

    auto_ptr<Plugin> p(load(key, rate));

    Plugin::FeatureSet f;

    Results r;

    float **data = 0;

    size_t channels = 0;

    size_t count = 100;

    if (!initAdapted(p.get(), channels, _step, _step, r)) return r;

    if (!data) data = createTestAudio(channels, _step, count);

    for (size_t i = 0; i < count; ++i) {

        float *ptr[channels];

        size_t idx = i * _step;

        for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;

        RealTime timestamp = RealTime::frame2RealTime(idx, rate);

        Plugin::FeatureSet fs = p->process(ptr, timestamp);

        appendFeatures(f, fs);

    }

    Plugin::FeatureSet fs = p->getRemainingFeatures();

    appendFeatures(f, fs);

    if (data) destroyTestAudio(data, channels);

    std::set<int> used;

    Plugin::OutputList outputs = p->getOutputDescriptors();

    for (Plugin::FeatureSet::const_iterator i = fs.begin();

         i != fs.end(); ++i) {

        int o = i->first;

        used.insert(o);

        if (o < 0 || o >= outputs.size()) {

            r.push_back(error("Data returned on nonexistent output"));

        }

    }

    for (int o = 0; o < outputs.size(); ++o) {

        if (used.find(o) == used.end()) {

            r.push_back(note("No results returned for one or more outputs"));       }

    }

    return r;

}

Test::Results

TestTimestamps::test(string key)

{

    int rate = 44100;

    auto_ptr<Plugin> p(load(key, rate));

    Plugin::FeatureSet f;

    Results r;

    float **data = 0;

    size_t channels = 0;

    size_t step = 0, block = 0;

    size_t count = 100;

    //!!! want to ensure buffer size adapter is not used:

    if (!initDefaults(p.get(), channels, step, block, r)) return r;

    if (!data) data = createTestAudio(channels, block, count);

    for (size_t i = 0; i < count; ++i) {

        float *ptr[channels];

        size_t idx = i * step;

        for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;

        RealTime timestamp = RealTime::frame2RealTime(idx, rate);

        Plugin::FeatureSet fs = p->process(ptr, timestamp);

        appendFeatures(f, fs);

    }

    Plugin::FeatureSet fs = p->getRemainingFeatures();

    appendFeatures(f, fs);

    if (data) destroyTestAudio(data, channels);

    Plugin::OutputList outputs = p->getOutputDescriptors();

    for (Plugin::FeatureSet::const_iterator i = fs.begin();

         i != fs.end(); ++i) {

        const Plugin::OutputDescriptor &o = outputs[i->first];

        const Plugin::FeatureList &fl = i->second;

        for (int j = 0; j < (int)fl.size(); ++j) {

            const Plugin::Feature &f = fl[j];

            switch (o.sampleType) {

            case Plugin::OutputDescriptor::OneSamplePerStep:

                if (f.hasTimestamp) {

                    r.push_back(note("Plugin returns features with timestamps on OneSamplePerStep output"));

                }

                if (f.hasDuration) {

                    r.push_back(note("Plugin returns features with durations on OneSamplePerStep output"));

                }

                break;

            case Plugin::OutputDescriptor::FixedSampleRate:

                break;

            case Plugin::OutputDescriptor::VariableSampleRate:

                if (!f.hasTimestamp) {

                    r.push_back(error("Plugin returns features with no timestamps on VariableSampleRate output"));

                }

                break;

            }

        }

    }

    return r;

}

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*

    Vamp Plugin Tester

    Chris Cannam, cannam@all-day-breakfast.com

    Centre for Digital Music, Queen Mary, University of London.

    Copyright 2009 QMUL.

    This program loads a Vamp plugin and tests its susceptibility to a

    number of common pitfalls, including handling of extremes of input

    data.  If you can think of any additional useful tests that are

    easily added, please send them to me.

    Permission is hereby granted, free of charge, to any person

    obtaining a copy of this software and associated documentation

    files (the "Software"), to deal in the Software without

    restriction, including without limitation the rights to use, copy,

    modify, merge, publish, distribute, sublicense, and/or sell copies

    of the Software, and to permit persons to whom the Software is

    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be

    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR

    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF

    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    Except as contained in this notice, the names of the Centre for

    Digital Music; Queen Mary, University of London; and Chris Cannam

    shall not be used in advertising or otherwise to promote the sale,

    use or other dealings in this Software without prior written

    authorization.

*/

#ifndef _TEST_OUTPUTS_H_

#define _TEST_OUTPUTS_H_

#include "Test.h"

#include "Tester.h"

class TestOutputNumbers : public Test

{

public:

    TestOutputNumbers() : Test() { }

    Results test(std::string key);

protected:

    static Tester::TestRegistrar<TestOutputNumbers> m_registrar;

};

class TestTimestamps : public Test

{

public:

    TestTimestamps() : Test() { }

    Results test(std::string key);

protected:

    static Tester::TestRegistrar<TestTimestamps> m_registrar;

};

#endif

        constructed and run "interleaved" with it (from same thread) - DONE

        called - DONE

            std::cout << " -- Performing test: " << i->first << std::endl;

                case Test::Result::Note:

                    std::cout << " ** NOTE: " << results[j].message() << std::endl;

                    break;

                    std::cout << " ** WARNING: " << results[j].message() << std::endl;

                    std::cout << " ** ERROR: " << results[j].message() << std::endl;

        std::cout << "ERROR: Failed to load plugin (key = \"" << m_key

            cout << "Testing plugin: " << keys[i] << endl;

                cout << name << ": All tests succeeded for this plugin" << endl;

                cout << name << ": Some tests failed for this plugin" << endl;

            cout << endl;

            cout << name << ": All tests succeeded" << endl;

            cout << name << ": Some tests failed" << endl;

            cout << name << ": All tests succeeded" << endl;

            cout << name << ": Some tests failed" << endl;