Mercurial > hg > vamp-plugin-tester

changeset 8:3019cb6b538d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
* Add nondeterministic and verbose options to control output * Dump returned features in a few more situations, if verbose is set * Dump features to stdout rather than stderr
author cannam
date Wed, 18 Mar 2009 10:51:30 +0000
parents 43eb3a4b95c8
children 7cc55187f5db
files Test.cpp Test.h TestDefaults.cpp TestDefaults.h TestInputExtremes.cpp TestInputExtremes.h TestMultipleRuns.cpp TestMultipleRuns.h TestOutputs.cpp TestOutputs.h TestStaticData.cpp TestStaticData.h Tester.cpp Tester.h vamp-plugin-tester.cpp
diffstat 15 files changed, 140 insertions(+), 75 deletions(-) [+]
line wrap: on
line diff
--- a/Test.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/Test.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -169,19 +169,19 @@
     for (Plugin::FeatureSet::const_iterator fsi = fs.begin();
          fsi != fs.end(); ++fsi) {
         int output = fsi->first;
-        std::cerr << "Output " << output << ":" << std::endl;
+        std::cout << "Output " << output << ":" << std::endl;
         const Plugin::FeatureList &fl = fsi->second;
         for (int i = 0; i < (int)fl.size(); ++i) {
-            std::cerr << "  Feature " << i << ":" << std::endl;
+            std::cout << "  Feature " << i << ":" << std::endl;
             const Plugin::Feature &f = fl[i];
-            std::cerr << "    Timestamp: " << (f.hasTimestamp ? "(none)" : f.timestamp.toText()) << std::endl;
-            std::cerr << "    Duration: " << (f.hasDuration ? "(none)" : f.duration.toText()) << std::endl;
-            std::cerr << "    Label: " << (f.label == "" ? "(none)" : f.label) << std::endl;
-            std::cerr << "    Value: " << (f.values.empty() ? "(none)" : "");
+            std::cout << "    Timestamp: " << (f.hasTimestamp ? "(none)" : f.timestamp.toText()) << std::endl;
+            std::cout << "    Duration: " << (f.hasDuration ? "(none)" : f.duration.toText()) << std::endl;
+            std::cout << "    Label: " << (f.label == "" ? "(none)" : f.label) << std::endl;
+            std::cout << "    Value: " << (f.values.empty() ? "(none)" : "");
             for (int j = 0; j < (int)f.values.size(); ++j) {
-                std::cerr << f.values[j] << " ";
+                std::cout << f.values[j] << " ";
             }
-            std::cerr << std::endl;
+            std::cout << std::endl;
         }
     }
 }
@@ -191,12 +191,12 @@
            const Plugin::FeatureSet &a,
            const Plugin::FeatureSet &b)
 {
-    std::cerr << r.message() << std::endl;
-    std::cerr << "\nFirst result set:" << std::endl;
+    std::cout << r.message() << std::endl;
+    std::cout << "\nFirst result set:" << std::endl;
     dump(a);
-    std::cerr << "\nSecond result set:" << std::endl;
+    std::cout << "\nSecond result set:" << std::endl;
     dump(b);
-    std::cerr << std::endl;
+    std::cout << std::endl;
 }
 
 bool
--- a/Test.h	Tue Mar 17 18:02:29 2009 +0000
+++ b/Test.h	Wed Mar 18 10:51:30 2009 +0000
@@ -48,17 +48,25 @@
 {
 public:
     virtual ~Test();
+
+    enum Option {
+        NoOption = 0x0,
+        NonDeterministic = 0x1,
+        Verbose = 0x2
+    };
+    typedef int Options;
     
     class Result {
 
     public:
         enum Code { Success, Note, Warning, Error };
 
+        Result() : m_code(Success) { }
         Result(Code c, std::string m) : m_code(c), m_message(m) { }
 
         Code code() const { return m_code; }
         std::string message() const { return m_message; }
-
+        
     protected:
         Code m_code;
         std::string m_message;
@@ -74,7 +82,7 @@
     class FailedToLoadPlugin { };
 
     // may throw FailedToLoadPlugin
-    virtual Results test(std::string key) = 0;
+    virtual Results test(std::string key, Options) = 0;
 
 protected:
     Test();
--- a/TestDefaults.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestDefaults.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -60,7 +60,7 @@
 static const size_t _step = 1000;
 
 Test::Results
-TestDefaultProgram::test(string key)
+TestDefaultProgram::test(string key, Options options)
 {
     Plugin::FeatureSet f[2];
     int rate = 44100;
@@ -91,8 +91,11 @@
     if (data) destroyTestAudio(data, channels);
 
     if (!(f[0] == f[1])) {
-        Result res = warning("Explicitly setting current program to its supposed current value changes the results");
-        dump(res, f[0], f[1]);
+        string message = "Explicitly setting current program to its supposed current value changes the results";
+        Result res;
+        if (options & NonDeterministic) res = note(message);
+        else res = error(message);
+        if (options & Verbose) dump(res, f[0], f[1]);
         r.push_back(res);
     } else {
         r.push_back(success());
@@ -102,7 +105,7 @@
 }
 
 Test::Results
-TestDefaultParameters::test(string key)
+TestDefaultParameters::test(string key, Options options)
 {
     Plugin::FeatureSet f[2];
     int rate = 44100;
@@ -139,8 +142,11 @@
     if (data) destroyTestAudio(data, channels);
 
     if (!(f[0] == f[1])) {
-        Result res = warning("Explicitly setting parameters to their supposed default values changes the results");
-        dump(res, f[0], f[1]);
+        string message = "Explicitly setting parameters to their supposed default values changes the results";
+        Result res;
+        if (options & NonDeterministic) res = note(message);
+        else res = error(message);
+        if (options & Verbose) dump(res, f[0], f[1]);
         r.push_back(res);
     } else {
         r.push_back(success());
@@ -150,7 +156,7 @@
 }
 
 Test::Results
-TestLengthyConstructor::test(string key)
+TestLengthyConstructor::test(string key, Options options)
 {
     time_t t0 = time(0);
     auto_ptr<Plugin> p(load(key));
--- a/TestDefaults.h	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestDefaults.h	Wed Mar 18 10:51:30 2009 +0000
@@ -47,7 +47,7 @@
 {
 public:
     TestDefaultProgram() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestDefaultProgram> m_registrar;
@@ -57,7 +57,7 @@
 {
 public:
     TestDefaultParameters() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestDefaultParameters> m_registrar;
@@ -67,7 +67,7 @@
 {
 public:
     TestLengthyConstructor() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestLengthyConstructor> m_registrar;
--- a/TestInputExtremes.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestInputExtremes.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -67,7 +67,7 @@
 TestRandomInput::m_registrar("C6 Random input");
 
 Test::Results
-TestNormalInput::test(string key)
+TestNormalInput::test(string key, Options options)
 {
     Plugin::FeatureSet f;
     int rate = 44100;
@@ -95,12 +95,13 @@
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
+        if (options & Verbose) dump(f);
     }
     return r;
 }
 
 Test::Results
-TestNoInput::test(string key)
+TestNoInput::test(string key, Options options)
 {
     auto_ptr<Plugin> p(load(key));
     Results r;
@@ -116,7 +117,7 @@
 }
 
 Test::Results
-TestShortInput::test(string key)
+TestShortInput::test(string key, Options options)
 {
     Plugin::FeatureSet f;
     int rate = 44100;
@@ -141,12 +142,13 @@
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
+        if (options & Verbose) dump(f);
     }
     return r;
 }
 
 Test::Results
-TestSilentInput::test(string key)
+TestSilentInput::test(string key, Options options)
 {
     Plugin::FeatureSet f;
     int rate = 44100;
@@ -172,12 +174,13 @@
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
+        if (options & Verbose) dump(f);
     }
     return r;
 }
 
 Test::Results
-TestTooLoudInput::test(string key)
+TestTooLoudInput::test(string key, Options options)
 {
     Plugin::FeatureSet f;
     int rate = 44100;
@@ -205,12 +208,13 @@
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
+        if (options & Verbose) dump(f);
     }
     return r;
 }
 
 Test::Results
-TestRandomInput::test(string key)
+TestRandomInput::test(string key, Options options)
 {
     Plugin::FeatureSet f;
     int rate = 44100;
@@ -238,6 +242,7 @@
         r.push_back(success());
     } else {
         r.push_back(warning("plugin returned one or more NaN/inf values"));
+        if (options & Verbose) dump(f);
     }
     return r;
 }
--- a/TestInputExtremes.h	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestInputExtremes.h	Wed Mar 18 10:51:30 2009 +0000
@@ -51,7 +51,7 @@
 {
 public:
     TestNormalInput() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
 
 protected:
     static Tester::TestRegistrar<TestNormalInput> m_registrar;
@@ -61,7 +61,7 @@
 {
 public:
     TestNoInput() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
 
 protected:
     static Tester::TestRegistrar<TestNoInput> m_registrar;
@@ -71,7 +71,7 @@
 {
 public:
     TestShortInput() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
 
 protected:
     static Tester::TestRegistrar<TestShortInput> m_registrar;
@@ -81,7 +81,7 @@
 {
 public:
     TestSilentInput() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
 
 protected:
     static Tester::TestRegistrar<TestSilentInput> m_registrar;
@@ -91,7 +91,7 @@
 {
 public:
     TestTooLoudInput() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
 
 protected:
     static Tester::TestRegistrar<TestTooLoudInput> m_registrar;
@@ -101,7 +101,7 @@
 {
 public:
     TestRandomInput() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
 
 protected:
     static Tester::TestRegistrar<TestRandomInput> m_registrar;
--- a/TestMultipleRuns.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestMultipleRuns.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -62,7 +62,7 @@
 static const size_t _step = 1000;
 
 Test::Results
-TestDistinctRuns::test(string key)
+TestDistinctRuns::test(string key, Options options)
 {
     Plugin::FeatureSet f[2];
     int rate = 44100;
@@ -89,8 +89,11 @@
     if (data) destroyTestAudio(data, channels);
 
     if (!(f[0] == f[1])) {
-        Result res = warning("Consecutive runs with separate instances produce different results");
-        dump(res, f[0], f[1]);
+        Result res;
+        string message = "Consecutive runs with separate instances produce different results";
+        if (options & NonDeterministic) res = note(message);
+        else res = error(message);
+        if (options & Verbose) dump(res, f[0], f[1]);
         r.push_back(res);
     } else {
         r.push_back(success());
@@ -100,7 +103,7 @@
 }
 
 Test::Results
-TestReset::test(string key)
+TestReset::test(string key, Options options)
 {
     Plugin::FeatureSet f[2];
     int rate = 44100;
@@ -129,8 +132,11 @@
     if (data) destroyTestAudio(data, channels);
 
     if (!(f[0] == f[1])) {
-        Result res = warning("Consecutive runs with the same instance (using reset) produce different results");
-        dump(res, f[0], f[1]);
+        string message = "Consecutive runs with the same instance (using reset) produce different results";
+        Result res;
+        if (options & NonDeterministic) res = note(message);
+        else res = error(message);
+        if (options & Verbose) dump(res, f[0], f[1]);
         r.push_back(res);
     } else {
         r.push_back(success());
@@ -140,7 +146,7 @@
 }
 
 Test::Results
-TestInterleavedRuns::test(string key)
+TestInterleavedRuns::test(string key, Options options)
 {
     Plugin::FeatureSet f[2];
     int rate = 44100;
@@ -178,8 +184,11 @@
     if (data) destroyTestAudio(data, channels);
 
     if (!(f[0] == f[1])) {
-        Result res = warning("Simultaneous runs with separate instances produce different results");
-        dump(res, f[0], f[1]);
+        string message = "Simultaneous runs with separate instances produce different results";
+        Result res;
+        if (options & NonDeterministic) res = note(message);
+        else res = error(message);
+        if (options & Verbose) dump(res, f[0], f[1]);
         r.push_back(res);
     } else {
         r.push_back(success());
@@ -189,7 +198,7 @@
 }
 
 Test::Results
-TestDifferentStartTimes::test(string key)
+TestDifferentStartTimes::test(string key, Options options)
 {
     Plugin::FeatureSet f[2];
     int rate = 44100;
@@ -217,8 +226,11 @@
     if (data) destroyTestAudio(data, channels);
 
     if (f[0] == f[1]) {
-        Result res = warning("Consecutive runs with different starting timestamps produce the same result");
-        dump(res, f[0], f[1]);
+        string message = "Consecutive runs with different starting timestamps produce the same result";
+        Result res;
+        if (options & NonDeterministic) res = note(message);
+        else res = error(message);
+        if (options & Verbose) dump(res, f[0], f[1]);
         r.push_back(res);
     } else {
         r.push_back(success());
--- a/TestMultipleRuns.h	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestMultipleRuns.h	Wed Mar 18 10:51:30 2009 +0000
@@ -47,7 +47,7 @@
 {
 public:
     TestDistinctRuns() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestDistinctRuns> m_registrar;
@@ -57,7 +57,7 @@
 {
 public:
     TestReset() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestReset> m_registrar;
@@ -67,7 +67,7 @@
 {
 public:
     TestInterleavedRuns() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestInterleavedRuns> m_registrar;
@@ -77,7 +77,7 @@
 {
 public:
     TestDifferentStartTimes() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestDifferentStartTimes> m_registrar;
--- a/TestOutputs.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestOutputs.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -59,7 +59,7 @@
 static const size_t _step = 1000;
 
 Test::Results
-TestOutputNumbers::test(string key)
+TestOutputNumbers::test(string key, Options options)
 {
     int rate = 44100;
     auto_ptr<Plugin> p(load(key, rate));
@@ -99,11 +99,12 @@
         }
     }
                 
+    if (!r.empty() && (options & Verbose)) dump(f);
     return r;
 }
 
 Test::Results
-TestTimestamps::test(string key)
+TestTimestamps::test(string key, Options options)
 {
     int rate = 44100;
 
@@ -159,5 +160,6 @@
         }
     }
 
+    if (!r.empty() && (options & Verbose)) dump(f);
     return r;
 }
--- a/TestOutputs.h	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestOutputs.h	Wed Mar 18 10:51:30 2009 +0000
@@ -47,7 +47,7 @@
 {
 public:
     TestOutputNumbers() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestOutputNumbers> m_registrar;
@@ -57,7 +57,7 @@
 {
 public:
     TestTimestamps() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestTimestamps> m_registrar;
--- a/TestStaticData.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestStaticData.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -57,7 +57,7 @@
 TestValueRanges::m_registrar("A3 Inappropriate value extents");
 
 Test::Results
-TestIdentifiers::test(string key)
+TestIdentifiers::test(string key, Options options)
 {
     auto_ptr<Plugin> p(load(key));
     
@@ -94,7 +94,7 @@
 }
 
 Test::Results
-TestEmptyFields::test(string key)
+TestEmptyFields::test(string key, Options options)
 {
     auto_ptr<Plugin> p(load(key));
 
@@ -147,7 +147,7 @@
 }
 
 Test::Results
-TestValueRanges::test(string key)
+TestValueRanges::test(string key, Options options)
 {
     auto_ptr<Plugin> p(load(key));
 
--- a/TestStaticData.h	Tue Mar 17 18:02:29 2009 +0000
+++ b/TestStaticData.h	Wed Mar 18 10:51:30 2009 +0000
@@ -47,7 +47,7 @@
 {
 public:
     TestIdentifiers() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     Result testIdentifier(std::string ident, std::string desc);
@@ -58,7 +58,7 @@
 {
 public:
     TestEmptyFields() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     Result testMandatory(std::string text, std::string desc);
@@ -70,7 +70,7 @@
 {
 public:
     TestValueRanges() : Test() { }
-    Results test(std::string key);
+    Results test(std::string key, Options options);
     
 protected:
     static Tester::TestRegistrar<TestValueRanges> m_registrar;
--- a/Tester.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/Tester.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -59,8 +59,9 @@
 
 using namespace std;
 
-Tester::Tester(std::string key) :
-    m_key(key)
+Tester::Tester(std::string key, Test::Options options) :
+    m_key(key),
+    m_options(options)
 {
 }
 
@@ -151,7 +152,7 @@
             std::cout << " -- Performing test: " << i->first << std::endl;
 
             Test *test = i->second->makeTest();
-            Test::Results results = test->test(m_key);
+            Test::Results results = test->test(m_key, m_options);
             delete test;
 
             set<string> printed;
--- a/Tester.h	Tue Mar 17 18:02:29 2009 +0000
+++ b/Tester.h	Wed Mar 18 10:51:30 2009 +0000
@@ -47,7 +47,7 @@
 class Tester
 {
 public:
-    Tester(std::string pluginKey);
+    Tester(std::string pluginKey, Test::Options);
     ~Tester();
 
     bool test(int &notes, int &warnings, int &errors);
@@ -71,6 +71,7 @@
     
 protected:
     std::string m_key;
+    Test::Options m_options;
     typedef std::map<std::string, Registrar *> Registry;
     static Registry &registry();
 };
--- a/vamp-plugin-tester.cpp	Tue Mar 17 18:02:29 2009 +0000
+++ b/vamp-plugin-tester.cpp	Wed Mar 18 10:51:30 2009 +0000
@@ -44,6 +44,7 @@
 #include <iostream>
 
 #include <cstdlib>
+#include <cstring>
 
 #include "Tester.h"
 
@@ -57,11 +58,16 @@
         "Copyright 2009 QMUL.\n"
         "Freely redistributable; published under a BSD-style license.\n\n"
         "Usage:\n"
-        "  " << name << " [<pluginbasename>:<plugin>]\n\n"
+        "  " << name << " [-n] [-v] [<pluginbasename>:<plugin>]\n\n"
         "Example:\n"
         "  " << name << " vamp-example-plugins:amplitudefollower\n\n"
         "With an argument, tests one plugin; without, tests all plugins in Vamp path.\n"
-        "If you have access to a runtime memory checker, you may find it especially\n"
+        "\nOptions:\n"
+        "  --nondeterministic, -n    Plugins may be nondeterministic: print a note\n"
+        "                            instead of an error if results differ between runs\n"
+        "  --verbose, -v             Show returned features each time a note, warning,\n"
+        "                            or error arises from feature data\n"
+        "\nIf you have access to a runtime memory checker, you may find it especially\n"
         "helpful to run this tester under it and watch for errors thus provoked.\n"
          << endl;
     exit(2);
@@ -76,20 +82,44 @@
         else ++scooter;
     }
     if (!name || !*name) name = argv[0];
+
+    bool nondeterministic = false;
+    bool verbose = false;
+    string argument;
+    for (int i = 1; i < argc; ++i) {
+        if (!argv[i]) break;
+        if (argv[i][0] == '-') {
+            if (!strcmp(argv[i], "-v") ||
+                !strcmp(argv[i], "--verbose")) {
+                verbose = 1;
+                continue;
+            }
+            if (!strcmp(argv[i], "-n") ||
+                !strcmp(argv[i], "--nondeterministic")) {
+                nondeterministic = 1;
+                continue;
+            }
+            usage(name);
+        } else {
+            if (argument != "") usage(name);
+            else argument = argv[i];
+        }
+    }
     
-    if (argc > 2) usage(name);
-    if (argc == 2 && argv[1][0] == '-') usage(name);
-
     cerr << name << ": Running..." << endl;
 
-    if (argc == 1) {
+    Test::Options opts = Test::NoOption;
+    if (nondeterministic) opts |= Test::NonDeterministic;
+    if (verbose) opts |= Test::Verbose;
+
+    if (argument == "") {
         bool good = true;
         Vamp::HostExt::PluginLoader::PluginKeyList keys =
             Vamp::HostExt::PluginLoader::getInstance()->listPlugins();
         int notes = 0, warnings = 0, errors = 0;
         for (int i = 0; i < (int)keys.size(); ++i) {
             cout << "Testing plugin: " << keys[i] << endl;
-            Tester tester(keys[i]);
+            Tester tester(keys[i], opts);
             if (tester.test(notes, warnings, errors)) {
                 cout << name << ": All tests succeeded for this plugin" << endl;
             } else {
@@ -115,8 +145,8 @@
             return 1;
         }   
     } else {
-        string key = argv[1];
-        Tester tester(key);
+        string key = argument;
+        Tester tester(key, opts);
         int notes = 0, warnings = 0, errors = 0;
         if (tester.test(notes, warnings, errors)) {
             cout << name << ": All tests succeeded";