Mercurial > hg > vamp-plugin-tester

changeset 0:f89128a316e7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
* Commit beginnings of a Vamp plugin test program
author cannam
date Thu, 12 Mar 2009 16:23:55 +0000
parents
children d7ef749300ed
files Makefile Test.cpp Test.h TestStaticData.cpp TestStaticData.h Tester.cpp Tester.h vamp-plugin-tester.cpp
diffstat 8 files changed, 881 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Makefile	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,25 @@
+
+LDFLAGS 	+= -lvamp-hostsdk -ldl
+CXXFLAGS	+= -Wall -Wextra
+
+OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o
+
+vamp-plugin-tester:	$(OBJECTS)
+
+clean:
+	rm -f $(OBJECTS)
+
+distclean:	clean
+	rm -f *~ vamp-plugin-tester
+
+depend:
+	makedepend -Y *.cpp *.h
+
+# DO NOT DELETE
+
+Test.o: Test.h
+TestStaticData.o: TestStaticData.h Test.h Tester.h
+Tester.o: Tester.h Test.h
+vamp-plugin-tester.o: Tester.h Test.h
+TestStaticData.o: Test.h Tester.h
+Tester.o: Test.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Test.cpp	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,105 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Fuzz Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#include "Test.h"
+
+#include <vamp-hostsdk/PluginLoader.h>
+
+using namespace Vamp;
+using namespace Vamp::HostExt;
+
+Test::Test() { }
+Test::~Test() { }
+
+Plugin *
+Test::load(std::string key, float rate)
+{
+    return PluginLoader::getInstance()->loadPlugin
+        (key, rate, PluginLoader::ADAPT_ALL);
+}
+
+void
+Test::appendFeatures(Plugin::FeatureSet &a, const Plugin::FeatureSet &b)
+{
+    for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {
+        int output = i->first;
+        const Plugin::FeatureList &fl = i->second;
+        Plugin::FeatureList &target = a[output];
+        for (Plugin::FeatureList::const_iterator j = fl.begin(); j != fl.end(); ++j) {
+            target.push_back(*j);
+        }
+    }
+}
+
+bool
+operator==(const Plugin::FeatureSet &a, const Plugin::FeatureSet &b)
+{
+    if (a.size() != b.size()) return false;
+    for (Plugin::FeatureSet::const_iterator ai = a.begin();
+         ai != a.end(); ++ai) {
+        int output = ai->first;
+        Plugin::FeatureSet::const_iterator bi = b.find(output);
+        if (bi == b.end()) return false;
+        if (!(ai->second == bi->second)) return false;
+    }
+    return true;
+}
+
+bool
+operator==(const Plugin::FeatureList &a, const Plugin::FeatureList &b)
+{
+    if (a.size() != b.size()) return false;
+    for (int i = 0; i < (int)a.size(); ++i) {
+        if (!(a[i] == b[i])) return false;
+    }
+    return true;
+}
+
+bool
+operator==(const Plugin::Feature &a, const Plugin::Feature &b)
+{
+    if (a.hasTimestamp != b.hasTimestamp) return false;
+    if (a.hasTimestamp && (a.timestamp != b.timestamp)) return false;
+    if (a.hasDuration != b.hasDuration) return false;
+    if (a.hasDuration && (a.duration != b.duration)) return false;
+    if (a.values != b.values) return false;
+    if (a.label != b.label) return false;
+    return true;
+}
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Test.h	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,96 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#ifndef _TEST_H_
+#define _TEST_H_
+
+#include <string>
+
+#include <vamp-hostsdk/Plugin.h>
+
+class Test
+{
+public:
+    virtual ~Test();
+    
+    class Result {
+
+    public:
+        enum Code { Success, Warning, Error };
+
+        Result(Code c, std::string m) : m_code(c), m_message(m) { }
+
+        Code code() { return m_code; }
+        std::string message() { return m_message; }
+
+    protected:
+        Code m_code;
+        std::string m_message;
+    };
+
+    static Result success() { return Result(Result::Success, ""); }
+    static Result warning(std::string m) { return Result(Result::Warning, m); }
+    static Result error(std::string m) { return Result(Result::Error, m); }
+
+    typedef std::vector<Result> Results;
+
+    class FailedToLoadPlugin { };
+
+    // may throw FailedToLoadPlugin
+    virtual Results test(std::string key) = 0;
+
+protected:
+    Test();
+
+    // may throw FailedToLoadPlugin
+    Vamp::Plugin *load(std::string key, float rate = 44100);
+
+    void appendFeatures(Vamp::Plugin::FeatureSet &a,
+                        const Vamp::Plugin::FeatureSet &b);
+};
+
+extern bool operator==(const Vamp::Plugin::FeatureSet &a,
+                       const Vamp::Plugin::FeatureSet &b);
+extern bool operator==(const Vamp::Plugin::FeatureList &a,
+                       const Vamp::Plugin::FeatureList &b);
+extern bool operator==(const Vamp::Plugin::Feature &a,
+                       const Vamp::Plugin::Feature &b);
+
+#endif
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestStaticData.cpp	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,201 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#include "TestStaticData.h"
+
+#include <vamp-hostsdk/Plugin.h>
+using namespace Vamp;
+
+#include <memory>
+using namespace std;
+
+#include <cmath>
+
+Tester::TestRegistrar<TestIdentifiers>
+TestIdentifiers::m_registrar("Invalid identifiers");
+
+Tester::TestRegistrar<TestEmptyFields>
+TestEmptyFields::m_registrar("Empty metadata fields");
+
+Tester::TestRegistrar<TestValueRanges>
+TestValueRanges::m_registrar("Inappropriate value extents");
+
+Test::Results
+TestIdentifiers::test(string key)
+{
+    auto_ptr<Plugin> p(load(key));
+    
+    Results r;
+    r.push_back(testIdentifier(p->getIdentifier(), "plugin identifier"));
+
+    Plugin::ParameterList params = p->getParameterDescriptors();
+    for (int i = 0; i < (int)params.size(); ++i) {
+        r.push_back(testIdentifier(params[i].identifier, "parameter identifier"));
+    }
+
+    Plugin::OutputList outputs = p->getOutputDescriptors();
+    for (int i = 0; i < (int)outputs.size(); ++i) {
+        r.push_back(testIdentifier(outputs[i].identifier, "output identifier"));
+    }
+
+    return r;
+}
+
+Test::Result
+TestIdentifiers::testIdentifier(string identifier, string desc)
+{
+    for (int i = 0; i < (int)identifier.length(); ++i) {
+        char c = identifier[i];
+        if (c >= 'a' && c <= 'z') continue;
+        if (c >= 'A' && c <= 'Z') continue;
+        if (c >= '0' && c <= '9') continue;
+        if (c == '_' || c == '-') continue;
+        return error
+            (desc + " \"" + identifier +
+             "\" contains invalid character(s); permitted are: [a-zA-Z0-9_-]");
+    }
+    return success();
+}
+
+Test::Results
+TestEmptyFields::test(string key)
+{
+    auto_ptr<Plugin> p(load(key));
+
+    Results r;
+
+    r.push_back(testMandatory(p->getName(), "plugin name"));
+    r.push_back(testRecommended(p->getDescription(), "plugin description"));
+    r.push_back(testRecommended(p->getMaker(), "plugin maker"));
+    r.push_back(testRecommended(p->getCopyright(), "plugin copyright"));
+    
+    Plugin::ParameterList params = p->getParameterDescriptors();
+    for (int i = 0; i < (int)params.size(); ++i) {
+        r.push_back(testMandatory
+                    (params[i].name,
+                     "plugin parameter \"" + params[i].identifier + "\" name"));
+        r.push_back(testRecommended
+                    (params[i].description,
+                     "plugin parameter \"" + params[i].identifier + "\" description"));
+    }
+    
+    Plugin::OutputList outputs = p->getOutputDescriptors();
+    for (int i = 0; i < (int)outputs.size(); ++i) {
+        r.push_back(testMandatory
+                    (outputs[i].name,
+                     "plugin output \"" + outputs[i].identifier + "\" name"));
+        r.push_back(testRecommended
+                    (outputs[i].description,
+                     "plugin output \"" + outputs[i].identifier + "\" description"));
+    }
+
+    return r;
+}
+
+Test::Result
+TestEmptyFields::testMandatory(string text, string desc)
+{
+    if (text == "") {
+        return error(desc + " is empty");
+    }
+    return success();
+}
+
+Test::Result
+TestEmptyFields::testRecommended(string text, string desc)
+{
+    if (text == "") {
+        return warning(desc + " is empty");
+    }
+    return success();
+}
+
+Test::Results
+TestValueRanges::test(string key)
+{
+    auto_ptr<Plugin> p(load(key));
+
+    Results r;
+
+    Plugin::ParameterList params = p->getParameterDescriptors();
+    for (int i = 0; i < (int)params.size(); ++i) {
+        Plugin::ParameterDescriptor &pd(params[i]);
+        string pfx("plugin parameter \"" + pd.identifier + "\"");
+        float min = pd.minValue;
+        float max = pd.maxValue;
+        float deft = pd.defaultValue;
+        if (max <= min) {
+            r.push_back(error(pfx + " maxValue <= minValue"));
+        }
+        if (deft < min || deft > max) {
+            r.push_back(error(pfx + " defaultValue out of range"));
+        }
+        if (pd.isQuantized) {
+            if (pd.quantizeStep == 0.f) {
+                r.push_back(error(pfx + " is quantized, but quantize step is zero"));
+            } else {
+
+                float epsilon = 0.00001f;
+                int qty = int((max - min) / pd.quantizeStep + 0.5);
+                float target = min + pd.quantizeStep * qty;
+                if (fabsf(max - target) > epsilon) {
+                    r.push_back(warning(pfx + " value range is not a multiple of quantize step"));
+                }
+
+                if (!pd.valueNames.empty()) {
+                    if ((int)pd.valueNames.size() < qty+1) {
+                        r.push_back(warning(pfx + " has fewer value names than quantize steps"));
+                    } else if ((int)pd.valueNames.size() > qty+1) {
+                        r.push_back(warning(pfx + " has more value names than quantize steps"));
+                    }
+                }
+
+                qty = int((deft - min) / pd.quantizeStep + 0.5);
+                target = min + pd.quantizeStep * qty;
+                if (fabsf(deft - target) > epsilon) {
+                    r.push_back(warning(pfx + " default value is not a multiple of quantize step beyond minimum"));
+                }
+            }
+        }
+    }
+
+    return r;
+}
+
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestStaticData.h	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,79 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#ifndef _TEST_STATIC_DATA_H_
+#define _TEST_STATIC_DATA_H_
+
+#include "Test.h"
+#include "Tester.h"
+
+class TestIdentifiers : public Test
+{
+public:
+    TestIdentifiers() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    Result testIdentifier(std::string ident, std::string desc);
+    static Tester::TestRegistrar<TestIdentifiers> m_registrar;
+};
+
+class TestEmptyFields : public Test
+{
+public:
+    TestEmptyFields() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    Result testMandatory(std::string text, std::string desc);
+    Result testRecommended(std::string text, std::string desc);
+    static Tester::TestRegistrar<TestEmptyFields> m_registrar;
+};
+
+class TestValueRanges : public Test
+{
+public:
+    TestValueRanges() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    static Tester::TestRegistrar<TestValueRanges> m_registrar;
+};
+
+#endif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Tester.cpp	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,177 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#include <vamp-hostsdk/PluginHostAdapter.h>
+#include <vamp-hostsdk/PluginInputDomainAdapter.h>
+#include <vamp-hostsdk/PluginLoader.h>
+
+#include <iostream>
+
+#include <cstring>
+#include <cstdlib>
+#include <cmath>
+
+#include "Tester.h"
+
+using Vamp::Plugin;
+using Vamp::PluginHostAdapter;
+using Vamp::RealTime;
+using Vamp::HostExt::PluginLoader;
+using Vamp::HostExt::PluginWrapper;
+using Vamp::HostExt::PluginInputDomainAdapter;
+
+using namespace std;
+
+Tester::Tester(std::string key) :
+    m_key(key)
+{
+}
+
+Tester::~Tester()
+{
+}
+
+Tester::Registry &
+Tester::registry()
+{
+    static Registry r;
+    return r;
+}
+
+bool
+Tester::test()
+{
+    /*
+      
+      Things I would like to see tested:
+
+      * Identifiers for parameters, outputs, or plugin itself contain
+        illegal characters - DONE
+
+      * Any of the plugin's name, maker etc fields are empty - DONE
+
+      * Plugin returns different results if another instance is
+        constructed and run "interleaved" with it (from same thread)
+ 
+      * Plugin's returned timestamps do not change as expected when
+        run with a different base timestamp for input (though there
+        could be legitimate reasons for this)
+
+      * Plugin fails when given zero-length or very short input
+
+      * Plugin fails when given "all digital zeros" input
+
+      * Plugin fails when given input that exceeds +/-1
+
+      * Plugin fails when given "normal" random input (just in case!)
+
+      * Plugin produces different results on second run, after reset
+        called
+
+      * Initial value of a parameter on plugin construction differs
+        from its default value (i.e. plugin produces different
+        results depending on whether parameter is set explicitly by
+        host to default value or not)
+
+      * Default value of a parameter is not quantized as specified - DONE
+
+      * Parameter minValue >= maxValue, or defaultValue < minValue
+        or > maxValue - DONE
+        
+      * If a plugin reports any programs, selecting default program
+        explicitly changes results (as for default parameters)
+
+      * Output feature does not hasTimestamp when output type is
+        VariableSampleRate
+
+      * Output feature hasTimestamp or hasDuration when output type is
+        OneSamplePerStep (warning only, this is not an error)
+
+      * Plugin fails gracelessly when constructed with "weird" sample
+        rate or initialised with "wrong" step size, block size, or
+        number of channels
+
+      * Plugin returns features whose output numbers do not have
+        a corresponding record in output descriptor list
+
+      * Plugin fails to return any features on some output (warning
+        only)
+
+      * Constructor takes a long time to run.  A fuzzy concept, but
+        suggests that some work should have been deferred to
+        initialise().  Warning only
+
+      Well, that's quite a lot of tests already.  What else?
+
+    */
+
+    bool good = true;
+
+    try {
+        for (Registry::const_iterator i = registry().begin();
+             i != registry().end(); ++i) {
+            
+            std::cerr << " -- Performing test: " << i->first << std::endl;
+
+            Test *test = i->second->makeTest();
+            Test::Results results = test->test(m_key);
+            delete test;
+            
+            for (int j = 0; j < (int)results.size(); ++j) {
+                switch (results[j].code()) {
+                case Test::Result::Success:
+                    break;
+                case Test::Result::Warning:
+                    std::cerr << " ** WARNING: " << results[j].message() << std::endl;
+                    break;
+                case Test::Result::Error:
+                    std::cerr << " ** ERROR: " << results[j].message() << std::endl;
+                    good = false;
+                    break;
+                }
+            }
+        }
+    } catch (Test::FailedToLoadPlugin) {
+        std::cerr << "ERROR: Failed to load plugin (key = \"" << m_key
+                  << "\")" << std::endl;
+    }
+
+    return good;
+}
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Tester.h	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,79 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#ifndef _TESTER_H_
+#define _TESTER_H_
+
+#include <string>
+
+#include "Test.h"
+
+class Tester
+{
+public:
+    Tester(std::string pluginKey);
+    ~Tester();
+
+    bool test();
+
+    class Registrar {
+    public:
+        Registrar(std::string name) { Tester::registerTest(name, this); }
+        virtual Test *makeTest() = 0;
+    };
+    
+    template <typename T>
+    class TestRegistrar : Registrar {
+    public:
+        TestRegistrar(std::string name) : Registrar(name) { }
+        virtual Test *makeTest() { return new T(); }
+    };
+
+    static void registerTest(std::string name, Registrar *r) {
+        registry()[name] = r;
+    }
+    
+protected:
+    std::string m_key;
+    typedef std::map<std::string, Registrar *> Registry;
+    static Registry &registry();
+};
+
+#endif
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/vamp-plugin-tester.cpp	Thu Mar 12 16:23:55 2009 +0000
@@ -0,0 +1,119 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#include <vamp-hostsdk/PluginHostAdapter.h>
+#include <vamp-hostsdk/PluginInputDomainAdapter.h>
+#include <vamp-hostsdk/PluginLoader.h>
+
+#include <iostream>
+
+#include <cstdlib>
+
+#include "Tester.h"
+
+using namespace std;
+
+void usage(const char *name)
+{
+    cerr << "\n"
+         << name << ": A Vamp plugin host that tests plugins for some likely errors.\n"
+        "Chris Cannam, Centre for Digital Music, Queen Mary, University of London.\n"
+        "Copyright 2009 QMUL.\n"
+        "Freely redistributable; published under a BSD-style license.\n\n"
+        "Usage:\n"
+        "  " << name << " [<pluginbasename>:<plugin>]\n\n"
+        "Example:\n"
+        "  " << name << " vamp-example-plugins:amplitudefollower\n\n"
+        "With an argument, tests one plugin; without, tests all plugins in Vamp path.\n"
+        "If you have access to a runtime memory checker, you may find it especially\n"
+        "helpful to run this tester under it and watch for errors thus provoked.\n"
+         << endl;
+    exit(2);
+}
+
+int main(int argc, char **argv)
+{
+    char *scooter = argv[0];
+    char *name = 0;
+    while (scooter && *scooter) {
+        if (*scooter == '/' || *scooter == '\\') name = ++scooter;
+        else ++scooter;
+    }
+    if (!name || !*name) name = argv[0];
+    
+    if (argc > 2) usage(name);
+    if (argc == 2 && argv[1][0] == '-') usage(name);
+
+    cerr << name << ": Running..." << endl;
+
+    if (argc == 1) {
+        bool good = true;
+        Vamp::HostExt::PluginLoader::PluginKeyList keys =
+            Vamp::HostExt::PluginLoader::getInstance()->listPlugins();
+        for (int i = 0; i < (int)keys.size(); ++i) {
+            cerr << "Testing plugin: " << keys[i] << endl;
+            Tester tester(keys[i]);
+            if (tester.test()) {
+                cerr << name << ": All tests succeeded for this plugin" << endl;
+            } else {
+                cerr << name << ": Some tests failed for this plugin" << endl;
+                good = false;
+            }
+            cerr << endl;
+        }
+        if (good) {
+            cerr << name << ": All tests succeeded" << endl;
+            return 0;
+        } else {
+            cerr << name << ": Some tests failed" << endl;
+            return 1;
+        }   
+    } else {
+        string key = argv[1];
+        Tester tester(key);
+        if (tester.test()) {
+            cerr << name << ": All tests succeeded" << endl;
+            return 0;
+        } else {
+            cerr << name << ": Some tests failed" << endl;
+            return 1;
+        }
+    }
+}
+