Mercurial > hg > vamp-plugin-tester

changeset 2:c9a4bd247497

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
* Add multiple-runs tests
author cannam
date Mon, 16 Mar 2009 17:43:31 +0000
parents d7ef749300ed
children 0f65bb22172b
files Makefile TestInputExtremes.cpp TestMultipleRuns.cpp TestMultipleRuns.h
diffstat 4 files changed, 298 insertions(+), 12 deletions(-) [+]
line wrap: on
line diff
--- a/Makefile	Thu Mar 12 17:09:53 2009 +0000
+++ b/Makefile	Mon Mar 16 17:43:31 2009 +0000
@@ -2,7 +2,7 @@
 LDFLAGS 	+= -lvamp-hostsdk -ldl
 CXXFLAGS	+= -Wall -Wextra
 
-OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o TestInputExtremes.o
+OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o TestInputExtremes.o TestMultipleRuns.o
 
 vamp-plugin-tester:	$(OBJECTS)
 
--- a/TestInputExtremes.cpp	Thu Mar 12 17:09:53 2009 +0000
+++ b/TestInputExtremes.cpp	Mon Mar 16 17:43:31 2009 +0000
@@ -45,6 +45,7 @@
 #include <memory>
 using namespace std;
 
+#include <cstdlib>
 #include <cmath>
 
 Tester::TestRegistrar<TestNormalInput>
@@ -68,6 +69,7 @@
 Test::Results
 TestNormalInput::test(string key)
 {
+    Plugin::FeatureSet f;
     int rate = 44100;
     auto_ptr<Plugin> p(load(key, rate));
     Results r;
@@ -83,11 +85,13 @@
             ++idx;
         }
         RealTime timestamp = RealTime::frame2RealTime(idx, rate);
-        p->process(block, timestamp);
+        Plugin::FeatureSet fs = p->process(block, timestamp);
+        appendFeatures(f, fs);
     }
     destroyBlock(block, channels);
     Plugin::FeatureSet fs = p->getRemainingFeatures();
-    if (allFeaturesValid(fs)) {
+    appendFeatures(f, fs);
+    if (allFeaturesValid(f)) {
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
@@ -114,6 +118,7 @@
 Test::Results
 TestShortInput::test(string key)
 {
+    Plugin::FeatureSet f;
     int rate = 44100;
     auto_ptr<Plugin> p(load(key, rate));
     Results r;
@@ -127,10 +132,12 @@
         }
         ++idx;
     }
-    p->process(block, RealTime::zeroTime);
+    Plugin::FeatureSet fs = p->process(block, RealTime::zeroTime);
+    appendFeatures(f, fs);
     destroyBlock(block, channels);
-    Plugin::FeatureSet fs = p->getRemainingFeatures();
-    if (allFeaturesValid(fs)) {
+    fs = p->getRemainingFeatures();
+    appendFeatures(f, fs);
+    if (allFeaturesValid(f)) {
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
@@ -141,6 +148,7 @@
 Test::Results
 TestSilentInput::test(string key)
 {
+    Plugin::FeatureSet f;
     int rate = 44100;
     auto_ptr<Plugin> p(load(key, rate));
     Results r;
@@ -154,11 +162,13 @@
     }
     for (int i = 0; i < 200; ++i) {
         RealTime timestamp = RealTime::frame2RealTime(i * blocksize, rate);
-        p->process(block, timestamp);
+        Plugin::FeatureSet fs = p->process(block, timestamp);
+        appendFeatures(f, fs);
     }
     destroyBlock(block, channels);
     Plugin::FeatureSet fs = p->getRemainingFeatures();
-    if (allFeaturesValid(fs)) {
+    appendFeatures(f, fs);
+    if (allFeaturesValid(f)) {
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
@@ -169,6 +179,7 @@
 Test::Results
 TestTooLoudInput::test(string key)
 {
+    Plugin::FeatureSet f;
     int rate = 44100;
     auto_ptr<Plugin> p(load(key, rate));
     Results r;
@@ -184,11 +195,13 @@
             ++idx;
         }
         RealTime timestamp = RealTime::frame2RealTime(idx, rate);
-        p->process(block, timestamp);
+        Plugin::FeatureSet fs = p->process(block, timestamp);
+        appendFeatures(f, fs);
     }
     destroyBlock(block, channels);
     Plugin::FeatureSet fs = p->getRemainingFeatures();
-    if (allFeaturesValid(fs)) {
+    appendFeatures(f, fs);
+    if (allFeaturesValid(f)) {
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
@@ -199,6 +212,7 @@
 Test::Results
 TestRandomInput::test(string key)
 {
+    Plugin::FeatureSet f;
     int rate = 44100;
     auto_ptr<Plugin> p(load(key, rate));
     Results r;
@@ -214,11 +228,13 @@
             ++idx;
         }
         RealTime timestamp = RealTime::frame2RealTime(idx, rate);
-        p->process(block, timestamp);
+        Plugin::FeatureSet fs = p->process(block, timestamp);
+        appendFeatures(f, fs);
     }
     destroyBlock(block, channels);
     Plugin::FeatureSet fs = p->getRemainingFeatures();
-    if (allFeaturesValid(fs)) {
+    appendFeatures(f, fs);
+    if (allFeaturesValid(f)) {
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestMultipleRuns.cpp	Mon Mar 16 17:43:31 2009 +0000
@@ -0,0 +1,194 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#include "TestMultipleRuns.h"
+
+#include <vamp-hostsdk/Plugin.h>
+using namespace Vamp;
+
+#include <memory>
+using namespace std;
+
+#include <cmath>
+
+Tester::TestRegistrar<TestDistinctRuns>
+TestDistinctRuns::m_registrar("Consecutive runs with separate instances");
+
+Tester::TestRegistrar<TestReset>
+TestReset::m_registrar("Consecutive runs with a single instance using reset");
+
+Tester::TestRegistrar<TestInterleavedRuns>
+TestInterleavedRuns::m_registrar("Simultaneous interleaved runs in a single thread");
+
+Test::Results
+TestDistinctRuns::test(string key)
+{
+    Plugin::FeatureSet f[2];
+    int rate = 44100;
+    Results r;
+    float **block = 0;
+    size_t channels = 0, step = 0, blocksize = 0;
+
+    for (int run = 0; run < 2; ++run) {
+        auto_ptr<Plugin> p(load(key, rate));
+        if (!initDefaults(p.get(), channels, step, blocksize, r)) return r;
+        if (!block) block = createBlock(channels, blocksize);
+        int idx = 0;
+        for (int i = 0; i < 100; ++i) {
+            for (size_t j = 0; j < blocksize; ++j) {
+                for (size_t c = 0; c < channels; ++c) {
+                    block[c][j] = sinf(float(idx) / 10.f);
+                    if ((i == 20 || i == 80) && (j < 2)) {
+                        block[c][j] = float(j) - 1.f;
+                    }
+                }
+                ++idx;
+            }
+            RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+            Plugin::FeatureSet fs = p->process(block, timestamp);
+            appendFeatures(f[run], fs);
+        }
+        Plugin::FeatureSet fs = p->getRemainingFeatures();
+        appendFeatures(f[run], fs);
+    }
+    if (block) destroyBlock(block, channels);
+
+    if (!(f[0] == f[1])) {
+        r.push_back(warning("Consecutive runs with separate instances produce different results"));
+    } else {
+        r.push_back(success());
+    }
+
+    return r;
+}
+
+Test::Results
+TestReset::test(string key)
+{
+    Plugin::FeatureSet f[2];
+    int rate = 44100;
+    Results r;
+    float **block = 0;
+    size_t channels = 0, step = 0, blocksize = 0;
+
+    auto_ptr<Plugin> p(load(key, rate));
+    for (int run = 0; run < 2; ++run) {
+        if (run == 1) p->reset();
+        if (!initDefaults(p.get(), channels, step, blocksize, r)) return r;
+        if (!block) block = createBlock(channels, blocksize);
+        int idx = 0;
+        for (int i = 0; i < 100; ++i) {
+            for (size_t j = 0; j < blocksize; ++j) {
+                for (size_t c = 0; c < channels; ++c) {
+                    block[c][j] = sinf(float(idx) / 10.f);
+                    if ((i == 20 || i == 80) && (j < 2)) {
+                        block[c][j] = float(j) - 1.f;
+                    }
+                }
+                ++idx;
+            }
+            RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+            Plugin::FeatureSet fs = p->process(block, timestamp);
+            appendFeatures(f[run], fs);
+        }
+        Plugin::FeatureSet fs = p->getRemainingFeatures();
+        appendFeatures(f[run], fs);
+    }
+    if (block) destroyBlock(block, channels);
+
+    if (!(f[0] == f[1])) {
+        r.push_back(warning("Consecutive runs with the same instance (using reset) produce different results"));
+    } else {
+        r.push_back(success());
+    }
+
+    return r;
+}
+
+Test::Results
+TestInterleavedRuns::test(string key)
+{
+    Plugin::FeatureSet f[2];
+    int rate = 44100;
+    Results r;
+    float **block = 0;
+    size_t channels = 0, step = 0, blocksize = 0;
+    Plugin *p[2];
+    for (int run = 0; run < 2; ++run) {
+        p[run] = load(key, rate);
+        if (!initDefaults(p[run], channels, step, blocksize, r)) {
+            delete p[run];
+            if (run > 0) delete p[0];
+            return r;
+        }
+        if (!block) block = createBlock(channels, blocksize);
+    }
+    int idx = 0;
+    for (int i = 0; i < 100; ++i) {
+        for (size_t j = 0; j < blocksize; ++j) {
+            for (size_t c = 0; c < channels; ++c) {
+                block[c][j] = sinf(float(idx) / 10.f);
+                if ((i == 20 || i == 80) && (j < 2)) {
+                    block[c][j] = float(j) - 1.f;
+                }
+            }
+            ++idx;
+        }
+        RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+        for (int run = 0; run < 2; ++run) {
+            Plugin::FeatureSet fs = p[run]->process(block, timestamp);
+            appendFeatures(f[run], fs);
+        }
+    }
+    for (int run = 0; run < 2; ++run) {
+        Plugin::FeatureSet fs = p[run]->getRemainingFeatures();
+        appendFeatures(f[run], fs);
+        delete p[run];
+    }
+
+    if (block) destroyBlock(block, channels);
+
+    if (!(f[0] == f[1])) {
+        r.push_back(warning("Consecutive runs with the same instance (using reset) produce different results"));
+    } else {
+        r.push_back(success());
+    }
+
+    return r;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestMultipleRuns.h	Mon Mar 16 17:43:31 2009 +0000
@@ -0,0 +1,76 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#ifndef _TEST_MULTIPLE_RUNS_H_
+#define _TEST_MULTIPLE_RUNS_H_
+
+#include "Test.h"
+#include "Tester.h"
+
+class TestDistinctRuns : public Test
+{
+public:
+    TestDistinctRuns() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    static Tester::TestRegistrar<TestDistinctRuns> m_registrar;
+};
+
+class TestReset : public Test
+{
+public:
+    TestReset() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    static Tester::TestRegistrar<TestReset> m_registrar;
+};
+
+class TestInterleavedRuns : public Test
+{
+public:
+    TestInterleavedRuns() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    static Tester::TestRegistrar<TestInterleavedRuns> m_registrar;
+};
+
+#endif