Mercurial > hg > vamp-plugin-tester

changeset 5:6a279da6fdd7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
* Add tests for default programs & parameters, etc
author cannam
date Tue, 17 Mar 2009 17:46:25 +0000
parents d8724c5a6d83
children ba3c8cc649d3
files Makefile TestDefaults.cpp TestDefaults.h TestMultipleRuns.cpp TestMultipleRuns.h TestOutputs.cpp Tester.cpp
diffstat 7 files changed, 273 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/Makefile	Tue Mar 17 17:11:35 2009 +0000
+++ b/Makefile	Tue Mar 17 17:46:25 2009 +0000
@@ -2,7 +2,7 @@
 LDFLAGS 	+= -lvamp-hostsdk -ldl
 CXXFLAGS	+= -Wall -Wextra
 
-OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o TestInputExtremes.o TestMultipleRuns.o TestOutputs.o
+OBJECTS		:= vamp-plugin-tester.o Tester.o Test.o TestStaticData.o TestInputExtremes.o TestMultipleRuns.o TestOutputs.o TestDefaults.o
 
 vamp-plugin-tester:	$(OBJECTS)
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestDefaults.cpp	Tue Mar 17 17:46:25 2009 +0000
@@ -0,0 +1,142 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#include "TestDefaults.h"
+
+#include <vamp-hostsdk/Plugin.h>
+using namespace Vamp;
+
+#include <memory>
+using namespace std;
+
+#include <cmath>
+
+Tester::TestRegistrar<TestDefaultProgram>
+TestDefaultProgram::m_registrar("E1 Inconsistent default program");
+
+Tester::TestRegistrar<TestDefaultParameters>
+TestDefaultParameters::m_registrar("E2 Inconsistent default parameters");
+
+static const size_t _step = 1000;
+
+Test::Results
+TestDefaultProgram::test(string key)
+{
+    Plugin::FeatureSet f[2];
+    int rate = 44100;
+    Results r;
+    float **data = 0;
+    size_t channels = 0;
+    size_t count = 100;
+
+    for (int run = 0; run < 2; ++run) {
+        auto_ptr<Plugin> p(load(key, rate));
+	if (p->getPrograms().empty()) return r;
+	if (run == 1) {
+            p->selectProgram(p->getCurrentProgram());
+        }
+        if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
+        if (!data) data = createTestAudio(channels, _step, count);
+        for (size_t i = 0; i < count; ++i) {
+            float *ptr[channels];
+            size_t idx = i * _step;
+            for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
+            RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+            Plugin::FeatureSet fs = p->process(ptr, timestamp);
+            appendFeatures(f[run], fs);
+        }
+        Plugin::FeatureSet fs = p->getRemainingFeatures();
+        appendFeatures(f[run], fs);
+    }
+    if (data) destroyTestAudio(data, channels);
+
+    if (!(f[0] == f[1])) {
+        r.push_back(warning("Explicitly setting current program to its supposed current value changes the results"));
+    } else {
+        r.push_back(success());
+    }
+
+    return r;
+}
+
+Test::Results
+TestDefaultParameters::test(string key)
+{
+    Plugin::FeatureSet f[2];
+    int rate = 44100;
+    Results r;
+    float **data = 0;
+    size_t channels = 0;
+    size_t count = 100;
+
+    for (int run = 0; run < 2; ++run) {
+        auto_ptr<Plugin> p(load(key, rate));
+	if (p->getParameterDescriptors().empty()) return r;
+	if (run == 1) {
+            Plugin::ParameterList pl = p->getParameterDescriptors();
+            for (int i = 0; i < (int)pl.size(); ++i) {
+                if (p->getParameter(pl[i].identifier) != pl[i].defaultValue) {
+                    r.push_back(error("Not all parameters have their default values when queried directly after construction"));
+                }
+                p->setParameter(pl[i].identifier, pl[i].defaultValue);
+            }
+        }
+        if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
+        if (!data) data = createTestAudio(channels, _step, count);
+        for (size_t i = 0; i < count; ++i) {
+            float *ptr[channels];
+            size_t idx = i * _step;
+            for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
+            RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+            Plugin::FeatureSet fs = p->process(ptr, timestamp);
+            appendFeatures(f[run], fs);
+        }
+        Plugin::FeatureSet fs = p->getRemainingFeatures();
+        appendFeatures(f[run], fs);
+    }
+    if (data) destroyTestAudio(data, channels);
+
+    if (!(f[0] == f[1])) {
+        r.push_back(warning("Explicitly setting parameters to their supposed default values changes the results"));
+    } else {
+        r.push_back(success());
+    }
+
+    return r;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TestDefaults.h	Tue Mar 17 17:46:25 2009 +0000
@@ -0,0 +1,67 @@
+/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
+
+/*
+    Vamp Plugin Tester
+    Chris Cannam, cannam@all-day-breakfast.com
+    Centre for Digital Music, Queen Mary, University of London.
+    Copyright 2009 QMUL.
+
+    This program loads a Vamp plugin and tests its susceptibility to a
+    number of common pitfalls, including handling of extremes of input
+    data.  If you can think of any additional useful tests that are
+    easily added, please send them to me.
+  
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use, copy,
+    modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
+    ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+    CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+    Except as contained in this notice, the names of the Centre for
+    Digital Music; Queen Mary, University of London; and Chris Cannam
+    shall not be used in advertising or otherwise to promote the sale,
+    use or other dealings in this Software without prior written
+    authorization.
+*/
+
+#ifndef _TEST_DEFAULTS_H_
+#define _TEST_DEFAULTS_H_
+
+#include "Test.h"
+#include "Tester.h"
+
+class TestDefaultProgram : public Test
+{
+public:
+    TestDefaultProgram() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    static Tester::TestRegistrar<TestDefaultProgram> m_registrar;
+};
+
+class TestDefaultParameters : public Test
+{
+public:
+    TestDefaultParameters() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    static Tester::TestRegistrar<TestDefaultParameters> m_registrar;
+};
+
+
+#endif
--- a/TestMultipleRuns.cpp	Tue Mar 17 17:11:35 2009 +0000
+++ b/TestMultipleRuns.cpp	Tue Mar 17 17:46:25 2009 +0000
@@ -56,6 +56,9 @@
 Tester::TestRegistrar<TestInterleavedRuns>
 TestInterleavedRuns::m_registrar("D3 Simultaneous interleaved runs in a single thread");
 
+Tester::TestRegistrar<TestDifferentStartTimes>
+TestDifferentStartTimes::m_registrar("D4 Consecutive runs with different start times");
+
 static const size_t _step = 1000;
 
 Test::Results
@@ -180,3 +183,40 @@
 
     return r;
 }
+
+Test::Results
+TestDifferentStartTimes::test(string key)
+{
+    Plugin::FeatureSet f[2];
+    int rate = 44100;
+    Results r;
+    float **data = 0;
+    size_t channels = 0;
+    size_t count = 100;
+
+    for (int run = 0; run < 2; ++run) {
+        auto_ptr<Plugin> p(load(key, rate));
+        if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
+        if (!data) data = createTestAudio(channels, _step, count);
+        for (size_t i = 0; i < count; ++i) {
+            float *ptr[channels];
+            size_t idx = i * _step;
+            for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
+            RealTime timestamp = RealTime::frame2RealTime(idx, rate);
+            if (run == 1) timestamp = timestamp + RealTime::fromSeconds(10);
+            Plugin::FeatureSet fs = p->process(ptr, timestamp);
+            appendFeatures(f[run], fs);
+        }
+        Plugin::FeatureSet fs = p->getRemainingFeatures();
+        appendFeatures(f[run], fs);
+    }
+    if (data) destroyTestAudio(data, channels);
+
+    if (f[0] == f[1]) {
+        r.push_back(warning("Consecutive runs with different starting timestamps produce the same result"));
+    } else {
+        r.push_back(success());
+    }
+
+    return r;
+}
--- a/TestMultipleRuns.h	Tue Mar 17 17:11:35 2009 +0000
+++ b/TestMultipleRuns.h	Tue Mar 17 17:46:25 2009 +0000
@@ -73,4 +73,14 @@
     static Tester::TestRegistrar<TestInterleavedRuns> m_registrar;
 };
 
+class TestDifferentStartTimes : public Test
+{
+public:
+    TestDifferentStartTimes() : Test() { }
+    Results test(std::string key);
+    
+protected:
+    static Tester::TestRegistrar<TestDifferentStartTimes> m_registrar;
+};
+
 #endif
--- a/TestOutputs.cpp	Tue Mar 17 17:11:35 2009 +0000
+++ b/TestOutputs.cpp	Tue Mar 17 17:46:25 2009 +0000
@@ -85,8 +85,8 @@
 
     std::set<int> used;
     Plugin::OutputList outputs = p->getOutputDescriptors();
-    for (Plugin::FeatureSet::const_iterator i = fs.begin();
-         i != fs.end(); ++i) {
+    for (Plugin::FeatureSet::const_iterator i = f.begin();
+         i != f.end(); ++i) {
         int o = i->first;
         used.insert(o);
         if (o < 0 || o >= (int)outputs.size()) {
@@ -133,25 +133,25 @@
     if (data) destroyTestAudio(data, channels);
 
     Plugin::OutputList outputs = p->getOutputDescriptors();
-    for (Plugin::FeatureSet::const_iterator i = fs.begin();
-         i != fs.end(); ++i) {
+    for (Plugin::FeatureSet::const_iterator i = f.begin();
+         i != f.end(); ++i) {
         const Plugin::OutputDescriptor &o = outputs[i->first];
         const Plugin::FeatureList &fl = i->second;
         for (int j = 0; j < (int)fl.size(); ++j) {
-            const Plugin::Feature &f = fl[j];
+            const Plugin::Feature &fe = fl[j];
             switch (o.sampleType) {
             case Plugin::OutputDescriptor::OneSamplePerStep:
-                if (f.hasTimestamp) {
+                if (fe.hasTimestamp) {
                     r.push_back(note("Plugin returns features with timestamps on OneSamplePerStep output"));
                 }
-                if (f.hasDuration) {
+                if (fe.hasDuration) {
                     r.push_back(note("Plugin returns features with durations on OneSamplePerStep output"));
                 }
                 break;
             case Plugin::OutputDescriptor::FixedSampleRate:
                 break;
             case Plugin::OutputDescriptor::VariableSampleRate:
-                if (!f.hasTimestamp) {
+                if (!fe.hasTimestamp) {
                     r.push_back(error("Plugin returns features with no timestamps on VariableSampleRate output"));
                 }
                 break;
--- a/Tester.cpp	Tue Mar 17 17:11:35 2009 +0000
+++ b/Tester.cpp	Tue Mar 17 17:46:25 2009 +0000
@@ -105,7 +105,7 @@
  
       * Plugin's returned timestamps do not change as expected when
         run with a different base timestamp for input (though there
-        could be legitimate reasons for this)
+        could be legitimate reasons for this) - DONE
 
       * Plugin produces different results on second run, after reset
         called - DONE
@@ -119,20 +119,20 @@
         explicitly changes results (as for default parameters)
 
       * Output feature does not hasTimestamp when output type is
-        VariableSampleRate
+        VariableSampleRate - DONE
 
       * Output feature hasTimestamp or hasDuration when output type is
-        OneSamplePerStep (warning only, this is not an error)
+        OneSamplePerStep (warning only, this is not an error) - DONE
 
       * Plugin fails gracelessly when constructed with "weird" sample
         rate or initialised with "wrong" step size, block size, or
         number of channels
 
       * Plugin returns features whose output numbers do not have
-        a corresponding record in output descriptor list
+        a corresponding record in output descriptor list - DONE
 
       * Plugin fails to return any features on some output (warning
-        only)
+        only) - DONE
 
       * Constructor takes a long time to run.  A fuzzy concept, but
         suggests that some work should have been deferred to