Mercurial > hg > vamp-plugin-tester

changeset 39:07144cdcbedf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Introduce logic for running only a single test (not yet wired up to command line interface)
author Chris Cannam
date Mon, 28 Jul 2014 11:11:59 +0100
parents f930285dfe5a
children 649f32c7eb41
files Test.h TestDefaults.cpp TestInitialise.cpp TestInputExtremes.cpp TestMultipleRuns.cpp TestOutputs.cpp TestStaticData.cpp Tester.cpp Tester.h vamp-plugin-tester.cpp
diffstat 10 files changed, 117 insertions(+), 63 deletions(-) [+]
line wrap: on
line diff
--- a/Test.h	Sat Jul 26 10:44:19 2014 +0100
+++ b/Test.h	Mon Jul 28 11:11:59 2014 +0100
@@ -50,9 +50,10 @@
     virtual ~Test();
 
     enum Option {
-        NoOption = 0x0,
-        NonDeterministic = 0x1,
-        Verbose = 0x2
+        NoOption           = 0x0,
+        NonDeterministic   = 0x1,
+        Verbose            = 0x2,
+        SingleTest         = 0x4
     };
     typedef int Options;
     
--- a/TestDefaults.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/TestDefaults.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -53,13 +53,13 @@
 #endif
 
 Tester::TestRegistrar<TestDefaultProgram>
-TestDefaultProgram::m_registrar("E1 Inconsistent default program");
+TestDefaultProgram::m_registrar("E1", "Inconsistent default program");
 
 Tester::TestRegistrar<TestDefaultParameters>
-TestDefaultParameters::m_registrar("E2 Inconsistent default parameters");
+TestDefaultParameters::m_registrar("E2", "Inconsistent default parameters");
 
 Tester::TestRegistrar<TestParametersOnReset>
-TestParametersOnReset::m_registrar("E3 Parameter retention through reset");
+TestParametersOnReset::m_registrar("E3", "Parameter retention through reset");
 
 static const size_t _step = 1000;
 
--- a/TestInitialise.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/TestInitialise.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -55,10 +55,10 @@
 #endif
 
 Tester::TestRegistrar<TestSampleRates>
-TestSampleRates::m_registrar("F1 Different sample rates");
+TestSampleRates::m_registrar("F1", "Different sample rates");
 
 Tester::TestRegistrar<TestLengthyConstructor>
-TestLengthyConstructor::m_registrar("F2 Lengthy constructor");
+TestLengthyConstructor::m_registrar("F2", "Lengthy constructor");
 
 static const size_t _step = 1000;
 
--- a/TestInputExtremes.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/TestInputExtremes.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -49,22 +49,22 @@
 #include <cmath>
 
 Tester::TestRegistrar<TestNormalInput>
-TestNormalInput::m_registrar("C1 Normal input");
+TestNormalInput::m_registrar("C1", "Normal input");
 
 Tester::TestRegistrar<TestNoInput>
-TestNoInput::m_registrar("C2 Empty input");
+TestNoInput::m_registrar("C2", "Empty input");
 
 Tester::TestRegistrar<TestShortInput>
-TestShortInput::m_registrar("C3 Short input");
+TestShortInput::m_registrar("C3", "Short input");
 
 Tester::TestRegistrar<TestSilentInput>
-TestSilentInput::m_registrar("C4 Absolutely silent input");
+TestSilentInput::m_registrar("C4", "Absolutely silent input");
 
 Tester::TestRegistrar<TestTooLoudInput>
-TestTooLoudInput::m_registrar("C5 Input beyond traditional +/-1 range");
+TestTooLoudInput::m_registrar("C5", "Input beyond traditional +/-1 range");
 
 Tester::TestRegistrar<TestRandomInput>
-TestRandomInput::m_registrar("C6 Random input");
+TestRandomInput::m_registrar("C6", "Random input");
 
 Test::Results
 TestNormalInput::test(string key, Options options)
--- a/TestMultipleRuns.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/TestMultipleRuns.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -52,16 +52,16 @@
 #endif
 
 Tester::TestRegistrar<TestDistinctRuns>
-TestDistinctRuns::m_registrar("D1 Consecutive runs with separate instances");
+TestDistinctRuns::m_registrar("D1", "Consecutive runs with separate instances");
 
 Tester::TestRegistrar<TestReset>
-TestReset::m_registrar("D2 Consecutive runs with a single instance using reset");
+TestReset::m_registrar("D2", "Consecutive runs with a single instance using reset");
 
 Tester::TestRegistrar<TestInterleavedRuns>
-TestInterleavedRuns::m_registrar("D3 Simultaneous interleaved runs in a single thread");
+TestInterleavedRuns::m_registrar("D3", "Simultaneous interleaved runs in a single thread");
 
 Tester::TestRegistrar<TestDifferentStartTimes>
-TestDifferentStartTimes::m_registrar("D4 Consecutive runs with different start times");
+TestDifferentStartTimes::m_registrar("D4", "Consecutive runs with different start times");
 
 static const size_t _step = 1000;
 
--- a/TestOutputs.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/TestOutputs.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -55,10 +55,10 @@
 #endif
 
 Tester::TestRegistrar<TestOutputNumbers>
-TestOutputNumbers::m_registrar("B1 Output number mismatching");
+TestOutputNumbers::m_registrar("B1", "Output number mismatching");
 
 Tester::TestRegistrar<TestTimestamps>
-TestTimestamps::m_registrar("B2 Invalid or dubious timestamp usage");
+TestTimestamps::m_registrar("B2", "Invalid or dubious timestamp usage");
 
 static const size_t _step = 1000;
 
--- a/TestStaticData.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/TestStaticData.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -50,16 +50,16 @@
 #include <cmath>
 
 Tester::TestRegistrar<TestIdentifiers>
-TestIdentifiers::m_registrar("A1 Invalid identifiers");
+TestIdentifiers::m_registrar("A1", "Invalid identifiers");
 
 Tester::TestRegistrar<TestEmptyFields>
-TestEmptyFields::m_registrar("A2 Empty metadata fields");
+TestEmptyFields::m_registrar("A2", "Empty metadata fields");
 
 Tester::TestRegistrar<TestValueRanges>
-TestValueRanges::m_registrar("A3 Inappropriate value extents");
+TestValueRanges::m_registrar("A3", "Inappropriate value extents");
 
 Tester::TestRegistrar<TestCategory>
-TestCategory::m_registrar("A3 Missing category");
+TestCategory::m_registrar("A3", "Missing category");
 
 Test::Results
 TestIdentifiers::test(string key, Options)
--- a/Tester.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/Tester.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -59,9 +59,10 @@
 
 using namespace std;
 
-Tester::Tester(std::string key, Test::Options options) :
+Tester::Tester(std::string key, Test::Options options, std::string singleTestId) :
     m_key(key),
-    m_options(options)
+    m_options(options),
+    m_singleTest(singleTestId)
 {
 }
 
@@ -69,6 +70,13 @@
 {
 }
 
+Tester::NameIndex &
+Tester::nameIndex()
+{
+    static NameIndex ix;
+    return ix;
+}
+
 Tester::Registry &
 Tester::registry()
 {
@@ -146,40 +154,30 @@
     bool good = true;
 
     try {
-        for (Registry::const_iterator i = registry().begin();
-             i != registry().end(); ++i) {
-            
-            std::cout << " -- Performing test: " << i->first << std::endl;
 
-            Test *test = i->second->makeTest();
-            Test::Results results = test->test(m_key, m_options);
-            delete test;
+        if (m_options & Test::SingleTest) {
 
-            set<string> printed;
-            
-            for (int j = 0; j < (int)results.size(); ++j) {
-                string message = results[j].message();
-                if (printed.find(message) != printed.end()) continue;
-                printed.insert(message);
-                switch (results[j].code()) {
-                case Test::Result::Success:
-                    break;
-                case Test::Result::Note:
-                    std::cout << " ** NOTE: " << results[j].message() << std::endl;
-                    ++notes;
-                    break;
-                case Test::Result::Warning:
-                    std::cout << " ** WARNING: " << results[j].message() << std::endl;
-                    ++warnings;
-                    break;
-                case Test::Result::Error:
-                    std::cout << " ** ERROR: " << results[j].message() << std::endl;
-                    ++errors;
-                    good = false;
-                    break;
-                }
+            if (registry().find(m_singleTest) != registry().end()) {
+
+                good = performTest(m_singleTest, notes, warnings, errors);
+
+            } else {
+
+                std::cout << " ** ERROR: Unknown single-test id \""
+                          << m_singleTest << "\"" << std::endl;
+                good = false;
+            }
+
+        } else {
+        
+            for (Registry::const_iterator i = registry().begin();
+                 i != registry().end(); ++i) {
+
+                bool thisGood = performTest(i->first, notes, warnings, errors);
+                if (!thisGood) good = false;
             }
         }
+
     } catch (Test::FailedToLoadPlugin) {
         std::cout << " ** ERROR: Failed to load plugin (key = \"" << m_key
                   << "\")" << std::endl;
@@ -194,3 +192,46 @@
     return good;
 }
 
+bool
+Tester::performTest(std::string id, int &notes, int &warnings, int &errors)
+{
+    std::cout << " -- Performing test: "
+              << id
+              << " "
+              << nameIndex()[id]
+              << std::endl;
+    
+    Test *test = registry()[id]->makeTest();
+    Test::Results results = test->test(m_key, m_options);
+    delete test;
+
+    set<string> printed;
+
+    bool good = true;
+
+    for (int j = 0; j < (int)results.size(); ++j) {
+        string message = results[j].message();
+        if (printed.find(message) != printed.end()) continue;
+        printed.insert(message);
+        switch (results[j].code()) {
+        case Test::Result::Success:
+            break;
+        case Test::Result::Note:
+            std::cout << " ** NOTE: " << results[j].message() << std::endl;
+            ++notes;
+            break;
+        case Test::Result::Warning:
+            std::cout << " ** WARNING: " << results[j].message() << std::endl;
+            ++warnings;
+            break;
+        case Test::Result::Error:
+            std::cout << " ** ERROR: " << results[j].message() << std::endl;
+            ++errors;
+            good = false;
+            break;
+        }
+    }
+
+    return good;
+}
+
--- a/Tester.h	Sat Jul 26 10:44:19 2014 +0100
+++ b/Tester.h	Mon Jul 28 11:11:59 2014 +0100
@@ -47,14 +47,16 @@
 class Tester
 {
 public:
-    Tester(std::string pluginKey, Test::Options);
+    Tester(std::string pluginKey, Test::Options, std::string singleTestId = "");
     ~Tester();
 
     bool test(int &notes, int &warnings, int &errors);
 
     class Registrar {
     public:
-        Registrar(std::string name) { Tester::registerTest(name, this); }
+        Registrar(std::string id, std::string name) { 
+            Tester::registerTest(id, name, this);
+        }
         virtual ~Registrar() { }
         virtual Test *makeTest() = 0;
     };
@@ -62,19 +64,26 @@
     template <typename T>
     class TestRegistrar : Registrar {
     public:
-        TestRegistrar(std::string name) : Registrar(name) { }
+        TestRegistrar(std::string id, std::string name) : 
+            Registrar(id, name) { }
         virtual Test *makeTest() { return new T(); }
     };
 
-    static void registerTest(std::string name, Registrar *r) {
-        registry()[name] = r;
+    static void registerTest(std::string id, std::string name, Registrar *r) {
+        nameIndex()[id] = name;
+        registry()[id] = r;
     }
     
 protected:
     std::string m_key;
     Test::Options m_options;
+    std::string m_singleTest;
+    typedef std::map<std::string, std::string> NameIndex;
     typedef std::map<std::string, Registrar *> Registry;
+    static NameIndex &nameIndex();
     static Registry &registry();
+
+    bool performTest(std::string id, int &notes, int &warnings, int &errors);
 };
 
 #endif
--- a/vamp-plugin-tester.cpp	Sat Jul 26 10:44:19 2014 +0100
+++ b/vamp-plugin-tester.cpp	Mon Jul 28 11:11:59 2014 +0100
@@ -118,9 +118,12 @@
 
     cerr << name << ": Running..." << endl;
 
+    std::string single = "";
+
     Test::Options opts = Test::NoOption;
     if (nondeterministic) opts |= Test::NonDeterministic;
     if (verbose) opts |= Test::Verbose;
+    if (single != "") opts |= Test::SingleTest;
 
     if (all) {
         bool good = true;
@@ -134,7 +137,7 @@
         int notes = 0, warnings = 0, errors = 0;
         for (int i = 0; i < (int)keys.size(); ++i) {
             cout << "Testing plugin: " << keys[i] << endl;
-            Tester tester(keys[i], opts);
+            Tester tester(keys[i], opts, single);
             if (tester.test(notes, warnings, errors)) {
                 cout << name << ": All tests succeeded for this plugin" << endl;
             } else {
@@ -161,7 +164,7 @@
         }   
     } else {
         string key = argument;
-        Tester tester(key, opts);
+        Tester tester(key, opts, single);
         int notes = 0, warnings = 0, errors = 0;
         if (tester.test(notes, warnings, errors)) {
             cout << name << ": All tests succeeded";