changeset 1:d7ef749300ed

* Add tests for extremes of input audio

--- a/Makefile	Thu Mar 12 16:23:55 2009 +0000
+++ b/Makefile	Thu Mar 12 17:09:53 2009 +0000
@@ -2,7 +2,7 @@
 LDFLAGS 	+= -lvamp-hostsdk -ldl
 CXXFLAGS	+= -Wall -Wextra
 
-OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o
+OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o TestInputExtremes.o
 
 vamp-plugin-tester:	$(OBJECTS)
 
@@ -18,6 +18,7 @@
 # DO NOT DELETE
 
 Test.o: Test.h
+TestInputExtremes.o: TestInputExtremes.h
 TestStaticData.o: TestStaticData.h Test.h Tester.h
 Tester.o: Tester.h Test.h
 vamp-plugin-tester.o: Tester.h Test.h

--- a/Test.cpp	Thu Mar 12 16:23:55 2009 +0000
+++ b/Test.cpp	Thu Mar 12 17:09:53 2009 +0000
@@ -44,6 +44,8 @@
 using namespace Vamp;
 using namespace Vamp::HostExt;
 
+#include <cmath>
+
 Test::Test() { }
 Test::~Test() { }
 
@@ -54,6 +56,44 @@
         (key, rate, PluginLoader::ADAPT_ALL);
 }
 
+float **
+Test::createBlock(size_t channels, size_t blocksize)
+{
+    float **b = new float *[channels];
+    for (size_t c = 0; c < channels; ++c) {
+        b[c] = new float[blocksize];
+    }
+    return b;
+}
+
+void
+Test::destroyBlock(float **blocks, size_t channels)
+{
+    for (size_t c = 0; c < channels; ++c) {
+        delete[] blocks[c];
+    }
+    delete[] blocks;
+}
+
+bool
+Test::initDefaults(Plugin *p, size_t &channels, size_t &step, size_t &block,
+                   Results &r)
+{
+    channels = p->getMinChannelCount();
+    block = p->getPreferredBlockSize();
+    step = p->getPreferredStepSize();
+    if (block == 0) block = 1024;
+    if (step == 0) {
+        if (p->getInputDomain() == Plugin::FrequencyDomain) step = block/2;
+        else step = block;
+    }
+    if (!p->initialise(channels, step, block)) {
+        r.push_back(error("initialisation with default values failed"));
+        return false;
+    }
+    return true;
+}
+
 void
 Test::appendFeatures(Plugin::FeatureSet &a, const Plugin::FeatureSet &b)
 {
@@ -68,6 +108,23 @@
 }
 
 bool
+Test::allFeaturesValid(const Plugin::FeatureSet &b)
+{
+    for (Plugin::FeatureSet::const_iterator i = b.begin(); i != b.end(); ++i) {
+        for (int j = 0; j < (int)i->second.size(); ++j) {
+            if (i->second[j].values.empty()) continue;
+            for (int k = 0; k < (int)i->second[j].values.size(); ++k) {
+                if (isnan(i->second[j].values[k]) ||
+                    isinf(i->second[j].values[k])) {
+                    return false;
+                }
+            }
+        }
+    }
+    return true;
+}
+
+bool
 operator==(const Plugin::FeatureSet &a, const Plugin::FeatureSet &b)
 {
     if (a.size() != b.size()) return false;

--- a/Test.h	Thu Mar 12 16:23:55 2009 +0000
+++ b/Test.h	Thu Mar 12 17:09:53 2009 +0000
@@ -81,8 +81,16 @@
     // may throw FailedToLoadPlugin
     Vamp::Plugin *load(std::string key, float rate = 44100);
 
+    float **createBlock(size_t channels, size_t blocksize);
+    void destroyBlock(float **blocks, size_t channels);
+
+    bool initDefaults(Vamp::Plugin *, size_t &channels,
+                      size_t &step, size_t &block, Results &r);
+
     void appendFeatures(Vamp::Plugin::FeatureSet &a,
                         const Vamp::Plugin::FeatureSet &b);
+
+    bool allFeaturesValid(const Vamp::Plugin::FeatureSet &); // i.e. no NaN/inf
 };
 
 extern bool operator==(const Vamp::Plugin::FeatureSet &a,

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestInputExtremes.cpp	Thu Mar 12 17:09:53 2009 +0000
@@ -0,0 +1,228 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, [e-mail]
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#include "TestInputExtremes.h"
+
+#include <vamp-hostsdk/Plugin.h>
+using namespace Vamp;
+
+#include <memory>
+using namespace std;
+
+#include <cmath>
+
+Tester::TestRegistrar<TestNormalInput>
+TestNormalInput::m_registrar("Normal input");
+
+Tester::TestRegistrar<TestNoInput>
+TestNoInput::m_registrar("Empty input");
+
+Tester::TestRegistrar<TestShortInput>
+TestShortInput::m_registrar("Short input");
+
+Tester::TestRegistrar<TestSilentInput>
+TestSilentInput::m_registrar("Absolutely silent input");
+
+Tester::TestRegistrar<TestTooLoudInput>
+TestTooLoudInput::m_registrar("Input beyond expected +/-1 range");
+
+Tester::TestRegistrar<TestRandomInput>
+TestRandomInput::m_registrar("Random input");
+
+Test::Results
+TestNormalInput::test(string key)
+{
+    int rate = 44100;
+    auto_ptr<Plugin> p(load(key, rate));
+    Results r;
+    size_t channels, step, blocksize;
+    if (!initDefaults(p.get(), channels, step, blocksize, r)) return r;
+    float **block = createBlock(channels, blocksize);
+    int idx = 0;
+    for (int i = 0; i < 200; ++i) {
+        for (size_t j = 0; j < blocksize; ++j) {
+            for (size_t c = 0; c < channels; ++c) {
+                block[c][j] = sinf(float(idx) / 10.f);
+            }
+            ++idx;
+        }
+        RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+        p->process(block, timestamp);
+    }
+    destroyBlock(block, channels);
+    Plugin::FeatureSet fs = p->getRemainingFeatures();
+    if (allFeaturesValid(fs)) {
+        r.push_back(success());
+    } else {
+        r.push_back(warning("plugin returned one or more NaN/inf values"));
+    }
+    return r;
+}
+
+Test::Results
+TestNoInput::test(string key)
+{
+    auto_ptr<Plugin> p(load(key));
+    Results r;
+    size_t channels, step, block;
+    if (!initDefaults(p.get(), channels, step, block, r)) return r;
+    Plugin::FeatureSet fs = p->getRemainingFeatures();
+    if (allFeaturesValid(fs)) {
+        r.push_back(success());
+    } else {
+        r.push_back(warning("plugin returned one or more NaN/inf values"));
+    }
+    return r;
+}
+
+Test::Results
+TestShortInput::test(string key)
+{
+    int rate = 44100;
+    auto_ptr<Plugin> p(load(key, rate));
+    Results r;
+    size_t channels, step, blocksize;
+    if (!initDefaults(p.get(), channels, step, blocksize, r)) return r;
+    float **block = createBlock(channels, blocksize);
+    int idx = 0;
+    for (size_t j = 0; j < blocksize; ++j) {
+        for (size_t c = 0; c < channels; ++c) {
+            block[c][j] = sinf(float(idx) / 10.f);
+        }
+        ++idx;
+    }
+    p->process(block, RealTime::zeroTime);
+    destroyBlock(block, channels);
+    Plugin::FeatureSet fs = p->getRemainingFeatures();
+    if (allFeaturesValid(fs)) {
+        r.push_back(success());
+    } else {
+        r.push_back(warning("plugin returned one or more NaN/inf values"));
+    }
+    return r;
+}
+
+Test::Results
+TestSilentInput::test(string key)
+{
+    int rate = 44100;
+    auto_ptr<Plugin> p(load(key, rate));
+    Results r;
+    size_t channels, step, blocksize;
+    if (!initDefaults(p.get(), channels, step, blocksize, r)) return r;
+    float **block = createBlock(channels, blocksize);
+    for (size_t j = 0; j < blocksize; ++j) {
+        for (size_t c = 0; c < channels; ++c) {
+            block[c][j] = 0.f;
+        }
+    }
+    for (int i = 0; i < 200; ++i) {
+        RealTime timestamp = RealTime::frame2RealTime(i * blocksize, rate);
+        p->process(block, timestamp);
+    }
+    destroyBlock(block, channels);
+    Plugin::FeatureSet fs = p->getRemainingFeatures();
+    if (allFeaturesValid(fs)) {
+        r.push_back(success());
+    } else {
+        r.push_back(warning("plugin returned one or more NaN/inf values"));
+    }
+    return r;
+}
+
+Test::Results
+TestTooLoudInput::test(string key)
+{
+    int rate = 44100;
+    auto_ptr<Plugin> p(load(key, rate));
+    Results r;
+    size_t channels, step, blocksize;
+    if (!initDefaults(p.get(), channels, step, blocksize, r)) return r;
+    float **block = createBlock(channels, blocksize);
+    int idx = 0;
+    for (int i = 0; i < 200; ++i) {
+        for (size_t j = 0; j < blocksize; ++j) {
+            for (size_t c = 0; c < channels; ++c) {
+                block[c][j] = 1000.f * sinf(float(idx) / 10.f);
+            }
+            ++idx;
+        }
+        RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+        p->process(block, timestamp);
+    }
+    destroyBlock(block, channels);
+    Plugin::FeatureSet fs = p->getRemainingFeatures();
+    if (allFeaturesValid(fs)) {
+        r.push_back(success());
+    } else {
+        r.push_back(warning("plugin returned one or more NaN/inf values"));
+    }
+    return r;
+}
+
+Test::Results
+TestRandomInput::test(string key)
+{
+    int rate = 44100;
+    auto_ptr<Plugin> p(load(key, rate));
+    Results r;
+    size_t channels, step, blocksize;
+    if (!initDefaults(p.get(), channels, step, blocksize, r)) return r;
+    float **block = createBlock(channels, blocksize);
+    int idx = 0;
+    for (int i = 0; i < 100; ++i) {
+        for (size_t j = 0; j < blocksize; ++j) {
+            for (size_t c = 0; c < channels; ++c) {
+                block[c][j] = float(drand48() * 2.0 - 1.0);
+            }
+            ++idx;
+        }
+        RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+        p->process(block, timestamp);
+    }
+    destroyBlock(block, channels);
+    Plugin::FeatureSet fs = p->getRemainingFeatures();
+    if (allFeaturesValid(fs)) {
+        r.push_back(success());
+    } else {
+        r.push_back(warning("plugin returned one or more NaN/inf values"));
+    }
+    return r;
+}
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestInputExtremes.h	Thu Mar 12 17:09:53 2009 +0000
@@ -0,0 +1,112 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, [e-mail]
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#ifndef _TEST_INPUT_EXTREMES_H_
+#define _TEST_INPUT_EXTREMES_H_
+
+#include "Test.h"
+#include "Tester.h"
+
+#include <string>
+
+#include <vamp-hostsdk/Plugin.h>
+
+class TestNormalInput : public Test
+{
+public:
+    TestNormalInput() : Test() { }
+    Results test(std::string key);
+
+protected:
+    static Tester::TestRegistrar<TestNormalInput> m_registrar;
+};
+
+class TestNoInput : public Test
+{
+public:
+    TestNoInput() : Test() { }
+    Results test(std::string key);
+
+protected:
+    static Tester::TestRegistrar<TestNoInput> m_registrar;
+};
+
+class TestShortInput : public Test
+{
+public:
+    TestShortInput() : Test() { }
+    Results test(std::string key);
+
+protected:
+    static Tester::TestRegistrar<TestShortInput> m_registrar;
+};
+
+class TestSilentInput : public Test
+{
+public:
+    TestSilentInput() : Test() { }
+    Results test(std::string key);
+
+protected:
+    static Tester::TestRegistrar<TestSilentInput> m_registrar;
+};
+
+class TestTooLoudInput : public Test
+{
+public:
+    TestTooLoudInput() : Test() { }
+    Results test(std::string key);
+
+protected:
+    static Tester::TestRegistrar<TestTooLoudInput> m_registrar;
+};
+
+class TestRandomInput : public Test
+{
+public:
+    TestRandomInput() : Test() { }
+    Results test(std::string key);
+
+protected:
+    static Tester::TestRegistrar<TestRandomInput> m_registrar;
+};
+
+
+#endif
+

--- a/Tester.cpp	Thu Mar 12 16:23:55 2009 +0000
+++ b/Tester.cpp	Thu Mar 12 17:09:53 2009 +0000
@@ -86,6 +86,19 @@
 
       * Any of the plugin's name, maker etc fields are empty - DONE
 
+      * Default value of a parameter is not quantized as specified - DONE
+
+      * Parameter minValue >= maxValue, or defaultValue < minValue
+        or > maxValue - DONE
+
+      * Plugin fails when given zero-length or very short input - DONE
+
+      * Plugin fails when given "all digital zeros" input - DONE
+
+      * Plugin fails when given input that exceeds +/-1 - DONE
+
+      * Plugin fails when given "normal" random input (just in case!) - DONE
+
       * Plugin returns different results if another instance is
         constructed and run "interleaved" with it (from same thread)
  
@@ -93,14 +106,6 @@
         run with a different base timestamp for input (though there
         could be legitimate reasons for this)
 
-      * Plugin fails when given zero-length or very short input
-
-      * Plugin fails when given "all digital zeros" input
-
-      * Plugin fails when given input that exceeds +/-1
-
-      * Plugin fails when given "normal" random input (just in case!)
-
       * Plugin produces different results on second run, after reset
         called
 
@@ -108,11 +113,6 @@
         from its default value (i.e. plugin produces different
         results depending on whether parameter is set explicitly by
         host to default value or not)
-
-      * Default value of a parameter is not quantized as specified - DONE
-
-      * Parameter minValue >= maxValue, or defaultValue < minValue
-        or > maxValue - DONE
         
       * If a plugin reports any programs, selecting default program
         explicitly changes results (as for default parameters)