Vamp Plugin Tester

configuration:

  - Release

platform:

  - x86

  - x64

install:

  - cinst --allow-empty-checksums smlnj

  - ps: '"[hostfingerprints]" | Out-File -Encoding "ASCII" -Append $env:USERPROFILE\mercurial.ini'

  - ps: '"code.soundsoftware.ac.uk = C7:27:7E:2C:1E:67:62:90:E9:8D:27:52:66:30:E6:FE:D4:2C:C5:30" | Out-File -Encoding "ASCII" -Append $env:USERPROFILE\mercurial.ini'

  - ps: '"[hostsecurity]" | Out-File -Encoding "ASCII" -Append $env:USERPROFILE\mercurial.ini'

  - ps: '"code.soundsoftware.ac.uk = code.soundsoftware.ac.uk:fingerprints=sha256:AF:67:D8:D6:D8:2F:28:FF:B9:1A:4D:CD:6A:93:25:EC:9E:47:9E:0E:E7:F1:FD:0B:97:B4:7C:D5:FA:2E:10:73" | Out-File -Encoding "ASCII" -Append $env:USERPROFILE\mercurial.ini'

before_build:

  - set PATH=%PATH%;C:\Program Files (x86)\SMLNJ\bin

  - repoint install

build:

  project: build\vamp-plugin-tester.sln

artifacts:

  - path: '**\*.exe'

syntax: glob

*.o

*~

*.so

*.dll

*.exe

vamp-plugin-tester

build/Release

build/x64/Release

build/Debug

build/x64/Debug

*.VC.db

*.VC.opendb

*.filters

glob:.repoint-*.bin

b1bc4d045a4be1dc802043d90c54538454fcb2d9 vamp-plugin-tester-v1.0

eff1772ba3974c163681387c1455bcc9aa853e1f vamp-plugin-tester-v1.0-erroneous

d19bbfc055abb6c7733f9f808b2bd1cd6dfa80c9 vamp-plugin-tester-v1.1

Changes in Vamp Plugin Tester v1.1 since the previous release v1.0:

Features

 * Add mechanism for listing the available tests and running only a

   single test at a time

 * Add test for reset() changing effective parameter values

 * Add test for fixed-sample-rate output defined without sample rate

 * Add test for existence of .cat file

 * Add test for boilerplate description text still in place for VamPy

   plugins

 * Make output more informative for some tests, and make some tests faster

Bug fixes

 * Fix erroneous verbose timestamp output for some tests

 * Don't complain when two consecutive runs with different starting

   times produce the same results, if neither set of results has any

   timestamps anyway

 * Link against pthread library on Linux, to support plugins with

   threading

    Permission is hereby granted, free of charge, to any person

    obtaining a copy of this software and associated documentation

    files (the "Software"), to deal in the Software without

    restriction, including without limitation the rights to use, copy,

    modify, merge, publish, distribute, sublicense, and/or sell copies

    of the Software, and to permit persons to whom the Software is

    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be

    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

    NONINFRINGEMENT. IN NO EVENT SHALL THE X CONSORTIUM BE LIABLE FOR

    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF

    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    Except as contained in this notice, the names of the Centre for

    Digital Music; Queen Mary, University of London; and Chris Cannam

    shall not be used in advertising or otherwise to promote the sale,

    use or other dealings in this Software without prior written

    authorization.

ARCHFLAGS	?=

LDFLAGS 	+= $(ARCHFLAGS) -ldl -pthread

CXXFLAGS	+= $(ARCHFLAGS) -std=c++98 -g -Wall -Wextra -Ivamp-plugin-sdk -pthread

# We include the Vamp Host SDK sources in the build here, so that we

# can build the entire thing with debug symbols even though the SDK

# would not normally have them

VAMP_SRCDIR	:= vamp-plugin-sdk/src/vamp-hostsdk

VAMP_OBJECTS	:= \

	$(VAMP_SRCDIR)/PluginHostAdapter.o \

	$(VAMP_SRCDIR)/RealTime.o \

	$(VAMP_SRCDIR)/PluginBufferingAdapter.o \

	$(VAMP_SRCDIR)/PluginChannelAdapter.o \

	$(VAMP_SRCDIR)/PluginInputDomainAdapter.o \

	$(VAMP_SRCDIR)/PluginLoader.o \

	$(VAMP_SRCDIR)/PluginSummarisingAdapter.o \

	$(VAMP_SRCDIR)/PluginWrapper.o \

	$(VAMP_SRCDIR)/Files.o \

	$(VAMP_SRCDIR)/acsymbols.o

OBJECTS		:= \

	$(VAMP_OBJECTS) \

	vamp-plugin-tester.o \

	Tester.o \

	Test.o \

	TestStaticData.o \

	TestInputExtremes.o \

	TestMultipleRuns.o \

	TestOutputs.o \

	TestDefaults.o \

	TestInitialise.o

vamp-plugin-tester:	vamp-plugin-sdk/README $(OBJECTS) $(VAMP_OBJECTS)

	$(CXX) $(OBJECTS) -o $@ $(LDFLAGS)

vamp-plugin-sdk/README:

	./repoint install

clean:

	rm -f $(OBJECTS) $(VAMP_OBJECTS)

distclean:	clean

	rm -f *~ vamp-plugin-tester

depend:

	makedepend -Y *.cpp *.h $(VAMP_SRCDIR)/*.cpp

# DO NOT DELETE

Test.o: Test.h

TestDefaults.o: TestDefaults.h Test.h Tester.h

Tester.o: Tester.h Test.h

TestInitialise.o: TestInitialise.h Test.h Tester.h

TestInputExtremes.o: TestInputExtremes.h Test.h Tester.h

TestMultipleRuns.o: TestMultipleRuns.h Test.h Tester.h

TestOutputs.o: TestOutputs.h Test.h Tester.h

TestStaticData.o: TestStaticData.h Test.h Tester.h

vamp-plugin-tester.o: Tester.h Test.h

TestDefaults.o: Test.h Tester.h

Tester.o: Test.h

TestInitialise.o: Test.h Tester.h

TestInputExtremes.o: Test.h Tester.h

TestMultipleRuns.o: Test.h Tester.h

TestOutputs.o: Test.h Tester.h

TestStaticData.o: Test.h Tester.h

vamp-plugin-sdk/src/vamp-hostsdk/PluginInputDomainAdapter.o: vamp-plugin-sdk/src/vamp-hostsdk/Window.h

vamp-plugin-sdk/src/vamp-hostsdk/PluginInputDomainAdapter.o: vamp-plugin-sdk/src/vamp-sdk/FFTimpl.cpp

vamp-plugin-sdk/src/vamp-hostsdk/RealTime.o: vamp-plugin-sdk/src/vamp-sdk/RealTime.cpp

Vamp Plugin Tester

==================

This program tests Vamp audio feature extraction plugins

(http://vamp-plugins.org/) for certain common failure cases.

To test a single plugin, run vamp-plugin-tester with the name of your

plugin library and plugin identifier, separated by a colon.  For example,

  $ vamp-plugin-tester vamp-example-plugins:amplitudefollower

The plugin library must be installed in the Vamp plugin path (you

cannot give the path to the library file).

Options

=======

Supply the -a or --all option to tell vamp-plugin-tester to test all

plugins found in your Vamp path.

Supply the -v or --verbose option to tell vamp-plugin-tester to print

out the whole content of its returned feature log for diagnostic

purposes each time it prints an error or warning that arises from the

contents of a returned feature.

Supply the -n or --nondeterministic option to tell vamp-plugin-tester

that your plugins are expected to return different results each time

they are run.  The default behaviour is to treat different results on

separate runs with the same input data as an error.

Supply the -t or --test option with a test ID argument to tell

vamp-plugin-tester to run only a single test, rather than the complete

test suite. To find out what test ID to use for a given test, run

vamp-plugin-tester with the --list-tests or -l option.

Errors and Warnings

===================

Each test may cause one or several notes, warnings, or errors to be

printed.  A note is printed when behaviour is observed that may be

correct behaviour but that is not always anticipated by the plugin

developer.  A warning is printed when behaviour is observed that is

technically legal but that in practice most often happens by mistake.

An error is printed when behaviour is observed that cannot be correct.

vamp-plugin-tester prints all of its commentary to the standard

output.  Standard error is usually used for diagnostic output printed

by the plugins themselves.

In addition to reports, vamp-plugin-tester runs some tests that are

intended to provoke the plugin into unexpected behaviour such as

memory errors.  If vamp-plugin-tester crashes during a test, this may

be why.  If you have access to a memory checker utility such as

valgrind, you are advised to run vamp-plugin-tester under it so as to

be informed of any memory errors that do not happen to cause crashes

(as well as memory leaks).  The vamp-plugin-tester binaries

distributed by QMUL have been compiled with debug information

included, in order to facilitate this type of use.

Error and Warning Reference

===========================

 ** ERROR: Failed to load plugin

 The plugin could not be loaded.  Remember that the plugin must be

 installed in the Vamp plugin path.

 Normally this message will be preceded by one of the following

 errors:

   Invalid plugin key <key> in loadPlugin

    - The argument given to vamp-plugin-tester could not be split

      into library name and plugin identifier.  Check the usage

      description above.

   No library found in Vamp path for plugin <key>

    - No Vamp plugin library of that name was found in the Vamp path.

      This message will often be accompanied by one of the following

      errors; if it isn't, then that probably means the file did not

      exist at all.

   Plugin <id> not found in library <name>

    - The library was found and loaded and was apparently a valid

      Vamp plugin library, but it didn't contain a plugin of that id.

      Check you typed the id correctly, and if this is your library,

      check that the vampGetPluginDescriptor function returns the

      plugin descriptor properly.

   Unable to load library <name>

    - A dynamic library of that name was found, but the system library

      loader could not load it.  Perhaps it depends on another library

      that is not available, or it was built for the wrong architecture.

      There may be more information in the error message.

   No vampGetPluginDescriptor function found in library <name>

    - A dynamic library of that name was found and loaded, but it

      lacked the necessary public vampGetPluginDescriptor function.

      * Are you sure this is a Vamp plugin library?

      * If you made it, did you remember to include the global

        vampGetPluginDescriptor function in your library along with

	your plugin classes?

      * If you are using Visual C++, did you remember to mark the

        vampGetPluginDescriptor symbol exported, as described in

        the README.msvc file in the SDK?

 ** ERROR: (plugin|parameter|output) identifier <x> contains invalid characters

 An identifier contains characters other than the permitted set (ASCII

 lower and upper case letters, digits, "-" and "_" only).

 ** ERROR: <field> is empty

 A mandatory field, such as the name of a parameter or output,

 contains no text.

 ** WARNING: <field> is empty

 An optional field, such as the description of a parameter or output,

 contains no text.

 ** ERROR: Plugin parameter <x> maxValue <= minValue

 The minimum and maximum values given for a parameter are equal or in

 the wrong order.

 ** ERROR: Plugin parameter <x> defaultValue out of range

 The default value for a parameter is not within the range defined by

 the minimum and maximum values for the parameter.

 ** ERROR: Plugin parameter <x> is quantized, but quantize step is zero

 The quantizeStep value in a parameter with isQuantized true is set to

 zero.

 ** WARNING: Plugin parameter <x> value range is not a multiple of quantize step

 A parameter's stated maximum value is not one of the possible values

 obtained by adding multiples of the quantize step on to the minimum

 value.

 ** WARNING: Plugin parameter <x> has (more|fewer) value names than quantize steps

 A quantized parameter lists some value names for its quantize steps,

 but not the right number.

 ** WARNING: Plugin parameter <x> default value is not a multiple of quantize

 step beyond minimum

 The default value for a parameter is not a value that the user could

 actually obtain, if only offered the quantized values to choose from.

 ** ERROR: Data returned on nonexistent output

 The output number key for a returned feature is outside the range of

 outputs listed in the plugin's output descriptor list.

 ** NOTE: No results returned for output <x>

 The plugin returned no features on one of its outputs, when given a

 simple test file.  This may be perfectly reasonable behaviour, but

 you might like to know about it.

 ** ERROR: Plugin output <x> has FixedSampleRate but gives sample rate as 0

 A plugin output that has a sample type of FixedSampleRate must have a

 non-zero sample rate. See

 https://code.soundsoftware.ac.uk/projects/vamp-plugin-sdk/wiki/SampleType

 ** NOTE: Plugin returns features with timestamps on OneSamplePerStep output

 ** NOTE: Plugin returns features with durations on OneSamplePerStep output

 Hosts will usually ignore timestamps and durations attached to any

 feature returned on a OneSamplePerStep output. See

 https://code.soundsoftware.ac.uk/projects/vamp-plugin-sdk/wiki/SampleType

 ** ERROR: Plugin returns features with no timestamps on VariableSampleRate output

 Timestamps are mandatory on all features associated with a

 VariableSampleRate output. See

 https://code.soundsoftware.ac.uk/projects/vamp-plugin-sdk/wiki/SampleType

 ** WARNING: Plugin returned one or more NaN/inf values

 The plugin returned features containing floating-point not-a-number

 or infinity values.  This warning may be associated with a test

 involving feeding some unexpected type of data to the plugin.

 ** ERROR: Consecutive runs with separate instances produce different results

 The plugin was constructed and run twice against the same input data,

 and returned different features each time.

 If you give the -n or --nondeterministic option, vamp-plugin-tester

 will downgrade this error to a note.

 ** ERROR: Consecutive runs with the same instance (using reset) produce different results

 The plugin was constructed, initialised, run against some input data,

 reset with a call to its reset() function, and run again against the

 same data; and it returned different features on each run.  This is

 often a sign of some simple error such as forgetting to implement

 reset().

 If you give the -n or --nondeterministic option, vamp-plugin-tester

 will downgrade this error to a note.

 ** ERROR: Simultaneous runs with separate instances produce different results

 Two instances of the plugin were constructed and run against the same

 input data, giving each block of data to one plugin's process call

 and then to the other's, "interleaving" the processing between the

 two instances (but within a single application thread); and the two

 instances returned different features.  This may indicate ill-advised

 use of static data shared between plugin instances.

 If you give the -n or --nondeterministic option, vamp-plugin-tester

 will downgrade this error to a note.

 ** WARNING: Consecutive runs with different starting timestamps produce the same result

 The plugin was run twice on the same audio data, but with different

 input timestamps, and it returned the same results each time.  While

 this is often unproblematic, it can indicate that a plugin failed to

 take the input timestamp into account when calculating its output

 timestamps (if any).

 If you give the -n or --nondeterministic option, vamp-plugin-tester

 will downgrade this warning to a note.

 ** ERROR: Explicitly setting current program to its supposed current value changes the results

 The plugin was constructed and run twice on the same data, once

 without changing its "program" setting, and again having set the

 program to the vaule returned by getCurrentProgram() (i.e. the same

 program that was supposed to be in effect already).  It returned

 different results for the two runs, suggesting that some internal

 data was changed in selectProgram in a way that differed from its

 default.

 If you give the -n or --nondeterministic option, vamp-plugin-tester

 will downgrade this error to a note.

 ** ERROR: Explicitly setting parameters to their supposed default values changes the results

 The plugin was constructed and run twice on the same data, once

 without changing any of its parameters, and again having set the

 parameters to their specified default values.  It returned different

 results for the two runs, suggesting that some internal data was

 changed when a parameter was set to its default, in a way that

 differed from the plugin's initially constructed state.

 If you give the -n or --nondeterministic option, vamp-plugin-tester

 will downgrade this error to a note.

 ** ERROR: Call to reset after setting parameters, but before processing, changes the results (parameter values not retained through reset?)

 The plugin was constructed and run twice on the same data. The first

 time, its parameters were set to some arbitrary values and it was

 initialised and run. The second time, its parameters were set to the

 same values and it was initialised, then reset(), then run. The two

 runs returned different results, suggesting that perhaps some

 parameter value was being modified within the reset() function. (This

 function should reset internal state within the plugin, but not

 parameter configuration.)

 If you give the -n or --nondeterministic option, vamp-plugin-tester

 will downgrade this error to a note.

 ** WARNING: Constructor takes some time to run: work should be deferred to initialise?

 The plugin took a long time to construct.  You should ensure that the

 constructor for the plugin runs as quickly as possible, because it

 may be called by a host that is only scanning the properties of all

 available plugins on startup.  Any serious initialisation work should

 be done in the initialise() function rather than the constructor.

Authors

=======

This program was written at the Centre for Digital Music at Queen

Mary, University of London, by Chris Cannam.  Copyright (c) 2009-2015 QMUL.

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*

    Vamp Plugin Fuzz Tester

    Chris Cannam, cannam@all-day-breakfast.com

    Centre for Digital Music, Queen Mary, University of London.

    Copyright 2009-2014 QMUL.

    This program loads a Vamp plugin and tests its susceptibility to a

    number of common pitfalls, including handling of extremes of input

    data.  If you can think of any additional useful tests that are

    easily added, please send them to me.

    Permission is hereby granted, free of charge, to any person

    obtaining a copy of this software and associated documentation

    files (the "Software"), to deal in the Software without

    restriction, including without limitation the rights to use, copy,

    modify, merge, publish, distribute, sublicense, and/or sell copies

    of the Software, and to permit persons to whom the Software is

    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be

    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR

    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF

    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    Except as contained in this notice, the names of the Centre for

    Digital Music; Queen Mary, University of London; and Chris Cannam

    shall not be used in advertising or otherwise to promote the sale,

    use or other dealings in this Software without prior written

    authorization.

*/

#include "Test.h"

#include <vamp-hostsdk/PluginLoader.h>

using namespace Vamp;

using namespace Vamp::HostExt;

#include <math.h>

#ifdef __SUNPRO_CC

#include <ieeefp.h>

#define isinf(x) (!finite(x))

#endif

Test::Test() { }

Test::~Test() { }

using std::cerr;

using std::cout;

using std::endl;

using std::string;

Plugin *

Test::load(string key, float rate)

{

    Plugin *p = PluginLoader::getInstance()->loadPlugin

        (key, rate, PluginLoader::ADAPT_ALL);

    if (!p) throw FailedToLoadPlugin();

    return p;

}

float **

Test::createBlock(size_t channels, size_t blocksize)

{

    float **b = new float *[channels];

    for (size_t c = 0; c < channels; ++c) {

        b[c] = new float[blocksize];

    }

    return b;

}

void

Test::destroyBlock(float **blocks, size_t channels)

{

    for (size_t c = 0; c < channels; ++c) {

        delete[] blocks[c];

    }

    delete[] blocks;

}

float **

Test::createTestAudio(size_t channels, size_t blocksize, size_t blocks)

{

    float **b = new float *[channels];

    for (size_t c = 0; c < channels; ++c) {

        b[c] = new float[blocksize * blocks];

        for (int i = 0; i < int(blocksize * blocks); ++i) {

            b[c][i] = sinf(float(i) / 10.f);

            if (i == 5005 || i == 20002) {

                b[c][i-2] = 0;

                b[c][i-1] = -1;

                b[c][i] = 1;

            }

        }

    }

    return b;

}

void

Test::destroyTestAudio(float **b, size_t channels)

{

    for (size_t c = 0; c < channels; ++c) {

        delete[] b[c];

    }

    delete[] b;

}

bool

Test::initDefaults(Plugin *p, size_t &channels, size_t &step, size_t &block,

                   Results &r)

{

    channels = p->getMinChannelCount();

    block = p->getPreferredBlockSize();

    step = p->getPreferredStepSize();

    if (block == 0) block = 1024;

    if (step == 0) {

        if (p->getInputDomain() == Plugin::FrequencyDomain) step = block/2;

        else step = block;

    }

    if (!p->initialise(channels, step, block)) {

        r.push_back(error("initialisation with default values failed"));

        return false;

    }

    return true;

}

bool

Test::initAdapted(Plugin *p, size_t &channels, size_t step, size_t block,

                  Results &r)

{

    channels = p->getMinChannelCount();

    if (!p->initialise(channels, step, block)) {

        r.push_back(error("initialisation failed"));

        return false;

    }

    return true;

}

void

Test::appendFeatures(Plugin::FeatureSet &a, const Plugin::FeatureSet &b)

{

    for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {

        int output = i->first;

        const Plugin::FeatureList &fl = i->second;

        Plugin::FeatureList &target = a[output];

        for (Plugin::FeatureList::const_iterator j = fl.begin(); j != fl.end(); ++j) {

            target.push_back(*j);

        }

    }

}

bool

Test::allFeaturesValid(const Plugin::FeatureSet &b)

{

    for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {

        for (int j = 0; j < (int)i->second.size(); ++j) {

            if (i->second[j].values.empty()) continue;

            for (int k = 0; k < (int)i->second[j].values.size(); ++k) {

                if (isnan(i->second[j].values[k]) ||

                    isinf(i->second[j].values[k])) {

                    return false;

                }

            }

        }

    }

    return true;

}

bool

Test::containsTimestamps(const Plugin::FeatureSet &b)

{

    for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {

        for (int j = 0; j < (int)i->second.size(); ++j) {

            if (i->second[j].values.empty()) continue;

            for (int k = 0; k < (int)i->second[j].values.size(); ++k) {

                if (i->second[j].hasTimestamp) {

                    return true;

                }

            }

        }

    }

    return false;

}

void

Test::dumpFeature(const Plugin::Feature &f, bool showValues,

                  const Plugin::Feature *other)

{

    cout << "    Timestamp: " << (!f.hasTimestamp ? "(none)" : f.timestamp.toText()) << endl;

    cout << "    Duration: " << (!f.hasDuration ? "(none)" : f.duration.toText()) << endl;

    cout << "    Label: " << (f.label == "" ? "(none)" : f.label) << endl;

    if (showValues) {

        cout << "    Values (" << f.values.size() << "): " << (f.values.empty() ? "(none)" : "");

        int n = f.values.size();

        if (!other) {

            for (int j = 0; j < n; ++j) {

                cout << f.values[j] << " ";

            }

        } else {

            int samecount = 0;

            int diffcount = 0;

            for (int j = 0; j <= n; ++j) {

                if (j < n && f.values[j] == other->values[j]) {

                    ++samecount;

                } else {

                    if (samecount > 0) {

                        cout << "(" << samecount << " identical) ";

                    }

                    samecount = 0;

                    if (j < n) {

                        ++diffcount;

                        if (diffcount > 20 && j + 10 < n) {

                            cout << "(remaining " << n - j << " values elided)";

                            break;

                        } else {

                            cout << f.values[j] << " [diff "

                                 << f.values[j] - other->values[j] << "] ";

                        }

                    }

                }

            }

        }

        cout << endl;

    } else {

        cout << "    Values (" << f.values.size() << "): (elided)" << endl;

    }

}

void

Test::dump(const Plugin::FeatureSet &fs, bool showValues)

{

    for (Plugin::FeatureSet::const_iterator fsi = fs.begin();

         fsi != fs.end(); ++fsi) {

        int output = fsi->first;

        cout << "Output " << output << ":" << endl;

        const Plugin::FeatureList &fl = fsi->second;

        for (int i = 0; i < (int)fl.size(); ++i) {

            cout << "  Feature " << i << ":" << endl;

            const Plugin::Feature &f = fl[i];

            dumpFeature(f, showValues);

        }

    }

}

void

Test::dumpTwo(const Result &r,

              const Plugin::FeatureSet &a,

              const Plugin::FeatureSet &b)

{

    std::cout << r.message() << std::endl;

    std::cout << "\nFirst result set:" << std::endl;

    dump(a, false);

    std::cout << "\nSecond result set:" << std::endl;

    dump(b, false);

    std::cout << std::endl;

}

void

Test::dumpDiff(const Result &r,

               const Plugin::FeatureSet &a,

               const Plugin::FeatureSet &b)

{

    cout << r.message() << endl;

    cout << "\nDifferences follow:" << endl;

    if (a.size() != b.size()) {

        cout << "*** First result set has features on " << a.size()

                  << " output(s), second has features on " << b.size()

                  << endl;

        return;

    }

    Plugin::FeatureSet::const_iterator ai = a.begin();

    Plugin::FeatureSet::const_iterator bi = b.begin();

    while (ai != a.end()) {

        if (ai->first != bi->first) {

            cout << "\n*** Output number mismatch: first result set says "

                      << ai->first << " where second says " << bi->first

                      << endl;

        } else {

            cout << "\nOutput " << ai->first << ":" << endl;

            if (ai->second.size() != bi->second.size()) {

                cout << "*** First result set has " << ai->second.size()

                          << " feature(s) on this output, second has "

                          << bi->second.size() << endl;

            } else {

                int fno = 0;

                int diffcount = 0;

                Plugin::FeatureList::const_iterator afi = ai->second.begin();

                Plugin::FeatureList::const_iterator bfi = bi->second.begin();

                while (afi != ai->second.end()) {

                    if (!(*afi == *bfi)) {

                        if (diffcount == 0) {

                            bool differInValues =

                                (afi->values.size() == bfi->values.size() &&

                                 afi->values != bfi->values);

                            if (afi->hasTimestamp != bfi->hasTimestamp) {

                                cout << "*** Feature " << fno << " differs in presence of timestamp (" << afi->hasTimestamp << " vs " << bfi->hasTimestamp << ")" << endl;

                            }

                            if (afi->hasTimestamp && (afi->timestamp != bfi->timestamp)) {

                                cout << "*** Feature " << fno << " differs in timestamp (" << afi->timestamp << " vs " << bfi->timestamp << " )" << endl;

                            }

                            if (afi->hasDuration != bfi->hasDuration) {

                                cout << "*** Feature " << fno << " differs in presence of duration (" << afi->hasDuration << " vs " << bfi->hasDuration << ")" << endl;

                            }

                            if (afi->hasDuration && (afi->duration != bfi->duration)) {

                                cout << "*** Feature " << fno << " differs in duration (" << afi->duration << " vs " << bfi->duration << " )" << endl;

                            }

                            if (afi->label != bfi->label) {

                                cout << "*** Feature " << fno << " differs in label" << endl;

                            }

                            if (afi->values.size() != bfi->values.size()) {

                                cout << "*** Feature " << fno << " differs in number of values (" << afi->values.size() << " vs " << bfi->values.size() << ")" << endl;

                            }

                            if (differInValues) {

                                cout << "*** Feature " << fno << " differs in values" << endl;

                            }

                            cout << "  First output:" << endl;

                            dumpFeature(*afi, differInValues);

                            cout << "  Second output:" << endl;

                            dumpFeature(*bfi, differInValues, &(*afi));

                        }

                        ++diffcount;

                    }

                    ++fno;

                    ++afi;

                    ++bfi;

                }

                if (diffcount > 1) {

                    cout << diffcount-1 << " subsequent differing feature(s) elided" << endl;

                }

            }

        }

        ++ai;

        ++bi;

    }

    cout << endl;

}

bool

operator==(const Plugin::FeatureSet &a, const Plugin::FeatureSet &b)

{

    if (a.size() != b.size()) return false;

    for (Plugin::FeatureSet::const_iterator ai = a.begin();

         ai != a.end(); ++ai) {

        int output = ai->first;

        Plugin::FeatureSet::const_iterator bi = b.find(output);

        if (bi == b.end()) return false;

        if (!(ai->second == bi->second)) return false;

    }

    return true;

}

bool

operator==(const Plugin::FeatureList &a, const Plugin::FeatureList &b)

{

    if (a.size() != b.size()) return false;

    for (int i = 0; i < (int)a.size(); ++i) {

        if (!(a[i] == b[i])) return false;

    }

    return true;

}

bool

operator==(const Plugin::Feature &a, const Plugin::Feature &b)

{

    if (a.hasTimestamp != b.hasTimestamp) return false;

    if (a.hasTimestamp && (a.timestamp != b.timestamp)) return false;

    if (a.hasDuration != b.hasDuration) return false;

    if (a.hasDuration && (a.duration != b.duration)) return false;

    if (a.values != b.values) return false;

    if (a.label != b.label) return false;

    return true;

}

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*

    Vamp Plugin Tester

    Chris Cannam, cannam@all-day-breakfast.com

    Centre for Digital Music, Queen Mary, University of London.

    Copyright 2009-2014 QMUL.

    This program loads a Vamp plugin and tests its susceptibility to a

    number of common pitfalls, including handling of extremes of input

    data.  If you can think of any additional useful tests that are

    easily added, please send them to me.

    Permission is hereby granted, free of charge, to any person

    obtaining a copy of this software and associated documentation

    files (the "Software"), to deal in the Software without

    restriction, including without limitation the rights to use, copy,

    modify, merge, publish, distribute, sublicense, and/or sell copies

    of the Software, and to permit persons to whom the Software is

    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be

    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR

    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF

    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    Except as contained in this notice, the names of the Centre for

    Digital Music; Queen Mary, University of London; and Chris Cannam

    shall not be used in advertising or otherwise to promote the sale,

    use or other dealings in this Software without prior written

    authorization.

*/

#ifndef _TEST_H_

#define _TEST_H_

#include <string>

#include <vamp-hostsdk/Plugin.h>

class Test

{

public:

    virtual ~Test();

    enum Option {

        NoOption           = 0x0,

        NonDeterministic   = 0x1,

        Verbose            = 0x2,

        SingleTest         = 0x4

    };

    typedef int Options;

    class Result {

    public:

        enum Code { Success, Note, Warning, Error };

        Result() : m_code(Success) { }

        Result(Code c, std::string m) : m_code(c), m_message(m) { }

        Code code() const { return m_code; }

        std::string message() const { return m_message; }

    protected:

        Code m_code;

        std::string m_message;

    };

    static Result success() { return Result(Result::Success, ""); }

    static Result note(std::string m) { return Result(Result::Note, m); }

    static Result warning(std::string m) { return Result(Result::Warning, m); }

    static Result error(std::string m) { return Result(Result::Error, m); }

    typedef std::vector<Result> Results;

    class FailedToLoadPlugin { };

    // may throw FailedToLoadPlugin

    virtual Results test(std::string key, Options) = 0;

protected:

    Test();

    // may throw FailedToLoadPlugin

    Vamp::Plugin *load(std::string key, float rate = 44100);

    float **createBlock(size_t channels, size_t blocksize);

    void destroyBlock(float **blocks, size_t channels);

    float **createTestAudio(size_t channels, size_t blocksize, size_t blocks);

    void destroyTestAudio(float **audio, size_t channels);

    // use plugin's preferred step/block size, return them:

    bool initDefaults(Vamp::Plugin *, size_t &channels,

                      size_t &step, size_t &block, Results &r);

    // use the given step/block size and an adapter:

    bool initAdapted(Vamp::Plugin *, size_t &channels,

                     size_t step, size_t block, Results &r);

    void appendFeatures(Vamp::Plugin::FeatureSet &a,

                        const Vamp::Plugin::FeatureSet &b);

    bool allFeaturesValid(const Vamp::Plugin::FeatureSet &); // i.e. no NaN/inf

    bool containsTimestamps(const Vamp::Plugin::FeatureSet &);

    void dumpFeature(const Vamp::Plugin::Feature &, bool showValues,

                     const Vamp::Plugin::Feature *other = 0);

    void dump(const Vamp::Plugin::FeatureSet &, bool showValues = true);

    void dumpTwo(const Result &r,

                 const Vamp::Plugin::FeatureSet &,

                 const Vamp::Plugin::FeatureSet &);

    void dumpDiff(const Result &r,

                  const Vamp::Plugin::FeatureSet &,

                  const Vamp::Plugin::FeatureSet &);

};

extern bool operator==(const Vamp::Plugin::FeatureSet &a,

                       const Vamp::Plugin::FeatureSet &b);

extern bool operator==(const Vamp::Plugin::FeatureList &a,

                       const Vamp::Plugin::FeatureList &b);

extern bool operator==(const Vamp::Plugin::Feature &a,

                       const Vamp::Plugin::Feature &b);

#endif

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */

/*

    Vamp Plugin Tester

    Chris Cannam, cannam@all-day-breakfast.com

    Centre for Digital Music, Queen Mary, University of London.

    Copyright 2009-2014 QMUL.

    This program loads a Vamp plugin and tests its susceptibility to a

    number of common pitfalls, including handling of extremes of input

    data.  If you can think of any additional useful tests that are

    easily added, please send them to me.

    Permission is hereby granted, free of charge, to any person

    obtaining a copy of this software and associated documentation

    files (the "Software"), to deal in the Software without

    restriction, including without limitation the rights to use, copy,

    modify, merge, publish, distribute, sublicense, and/or sell copies

    of the Software, and to permit persons to whom the Software is

    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be

    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR

    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF

    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION

    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

    Except as contained in this notice, the names of the Centre for

    Digital Music; Queen Mary, University of London; and Chris Cannam

    shall not be used in advertising or otherwise to promote the sale,

    use or other dealings in this Software without prior written

    authorization.

*/

#include "TestDefaults.h"

#include <vamp-hostsdk/Plugin.h>

using namespace Vamp;

#include <memory>

using namespace std;

#include <cmath>

#include <time.h>

Tester::TestRegistrar<TestDefaultProgram>

TestDefaultProgram::m_registrar("E1", "Inconsistent default program");

Tester::TestRegistrar<TestDefaultParameters>

TestDefaultParameters::m_registrar("E2", "Inconsistent default parameters");

Tester::TestRegistrar<TestParametersOnReset>

TestParametersOnReset::m_registrar("E3", "Parameter retention through reset");

static const size_t _step = 1000;

Test::Results

TestDefaultProgram::test(string key, Options options)

{

    Plugin::FeatureSet f[2];

    int rate = 44100;

    Results r;

    float **data = 0;

    size_t channels = 0;

    size_t count = 100;

    for (int run = 0; run < 2; ++run) {

        auto_ptr<Plugin> p(load(key, rate));

	if (p->getPrograms().empty()) return r;

	if (run == 1) {

            p->selectProgram(p->getCurrentProgram());

        }

        if (!initAdapted(p.get(), channels, _step, _step, r)) return r;

        if (!data) data = createTestAudio(channels, _step, count);

        for (size_t i = 0; i < count; ++i) {

            float **ptr = new float *[channels];

            size_t idx = i * _step;

            for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;

            RealTime timestamp = RealTime::frame2RealTime(idx, rate);

            Plugin::FeatureSet fs = p->process(ptr, timestamp);

            delete[] ptr;

            appendFeatures(f[run], fs);

        }

        Plugin::FeatureSet fs = p->getRemainingFeatures();

        appendFeatures(f[run], fs);

    }

    if (data) destroyTestAudio(data, channels);

    if (!(f[0] == f[1])) {

        string message = "Explicitly setting current program to its supposed current value changes the results";

        Result res;

        if (options & NonDeterministic) res = note(message);

        else res = error(message);

        if (options & Verbose) dumpDiff(res, f[0], f[1]);

        r.push_back(res);

    } else {

        r.push_back(success());

    }

    return r;

}

Test::Results

TestDefaultParameters::test(string key, Options options)

{

    Plugin::FeatureSet f[2];

    int rate = 44100;

    Results r;

    float **data = 0;

    size_t channels = 0;

    size_t count = 100;

    for (int run = 0; run < 2; ++run) {

        auto_ptr<Plugin> p(load(key, rate));

	if (p->getParameterDescriptors().empty()) return r;

	if (run == 1) {

            Plugin::ParameterList pl = p->getParameterDescriptors();

            for (int i = 0; i < (int)pl.size(); ++i) {