annotate Tester.cpp @ 80:f84e12a1553c tip

Update server certificate fingerprints
author Chris Cannam
date Wed, 14 Aug 2019 14:58:48 +0100
parents f1e8e14e9c96
children
rev   line source
cannam@0 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
cannam@0 2
cannam@0 3 /*
cannam@0 4 Vamp Plugin Tester
cannam@0 5 Chris Cannam, cannam@all-day-breakfast.com
cannam@0 6 Centre for Digital Music, Queen Mary, University of London.
Chris@42 7 Copyright 2009-2014 QMUL.
cannam@0 8
cannam@0 9 This program loads a Vamp plugin and tests its susceptibility to a
cannam@0 10 number of common pitfalls, including handling of extremes of input
cannam@0 11 data. If you can think of any additional useful tests that are
cannam@0 12 easily added, please send them to me.
cannam@0 13
cannam@0 14 Permission is hereby granted, free of charge, to any person
cannam@0 15 obtaining a copy of this software and associated documentation
cannam@0 16 files (the "Software"), to deal in the Software without
cannam@0 17 restriction, including without limitation the rights to use, copy,
cannam@0 18 modify, merge, publish, distribute, sublicense, and/or sell copies
cannam@0 19 of the Software, and to permit persons to whom the Software is
cannam@0 20 furnished to do so, subject to the following conditions:
cannam@0 21
cannam@0 22 The above copyright notice and this permission notice shall be
cannam@0 23 included in all copies or substantial portions of the Software.
cannam@0 24
cannam@0 25 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
cannam@0 26 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
cannam@0 27 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
cannam@0 28 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
cannam@0 29 ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
cannam@0 30 CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
cannam@0 31 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
cannam@0 32
cannam@0 33 Except as contained in this notice, the names of the Centre for
cannam@0 34 Digital Music; Queen Mary, University of London; and Chris Cannam
cannam@0 35 shall not be used in advertising or otherwise to promote the sale,
cannam@0 36 use or other dealings in this Software without prior written
cannam@0 37 authorization.
cannam@0 38 */
cannam@0 39
cannam@0 40 #include <vamp-hostsdk/PluginHostAdapter.h>
cannam@0 41 #include <vamp-hostsdk/PluginInputDomainAdapter.h>
cannam@0 42 #include <vamp-hostsdk/PluginLoader.h>
cannam@0 43
cannam@0 44 #include <iostream>
cannam@0 45
cannam@0 46 #include <cstring>
cannam@0 47 #include <cstdlib>
cannam@0 48 #include <cmath>
cannam@4 49 #include <set>
cannam@0 50
cannam@0 51 #include "Tester.h"
cannam@0 52
cannam@0 53 using Vamp::Plugin;
cannam@0 54 using Vamp::PluginHostAdapter;
cannam@0 55 using Vamp::RealTime;
cannam@0 56 using Vamp::HostExt::PluginLoader;
cannam@0 57 using Vamp::HostExt::PluginWrapper;
cannam@0 58 using Vamp::HostExt::PluginInputDomainAdapter;
cannam@0 59
cannam@0 60 using namespace std;
cannam@0 61
Chris@39 62 Tester::Tester(std::string key, Test::Options options, std::string singleTestId) :
cannam@8 63 m_key(key),
Chris@39 64 m_options(options),
Chris@39 65 m_singleTest(singleTestId)
cannam@0 66 {
cannam@0 67 }
cannam@0 68
cannam@0 69 Tester::~Tester()
cannam@0 70 {
cannam@0 71 }
cannam@0 72
Chris@39 73 Tester::NameIndex &
Chris@39 74 Tester::nameIndex()
Chris@39 75 {
Chris@39 76 static NameIndex ix;
Chris@39 77 return ix;
Chris@39 78 }
Chris@39 79
cannam@0 80 Tester::Registry &
cannam@0 81 Tester::registry()
cannam@0 82 {
cannam@0 83 static Registry r;
cannam@0 84 return r;
cannam@0 85 }
cannam@0 86
Chris@40 87 void
Chris@40 88 Tester::listTests()
Chris@40 89 {
Chris@40 90 cout << endl;
Chris@40 91 cout << "Total tests: " << nameIndex().size() << "\n" << endl;
Chris@40 92 cout << "ID | Name" << endl;
Chris@40 93 cout << "---+-----" << endl;
Chris@40 94 for (NameIndex::const_iterator i = nameIndex().begin();
Chris@40 95 i != nameIndex().end(); ++i) {
Chris@40 96 cout << i->first << " | " << i->second << endl;
Chris@40 97 }
Chris@40 98 cout << endl;
Chris@40 99 }
Chris@40 100
cannam@0 101 bool
cannam@4 102 Tester::test(int &notes, int &warnings, int &errors)
cannam@0 103 {
cannam@0 104 /*
cannam@0 105
cannam@0 106 Things I would like to see tested:
cannam@0 107
cannam@0 108 * Identifiers for parameters, outputs, or plugin itself contain
cannam@0 109 illegal characters - DONE
cannam@0 110
cannam@0 111 * Any of the plugin's name, maker etc fields are empty - DONE
cannam@0 112
cannam@1 113 * Default value of a parameter is not quantized as specified - DONE
cannam@1 114
cannam@1 115 * Parameter minValue >= maxValue, or defaultValue < minValue
cannam@1 116 or > maxValue - DONE
cannam@1 117
cannam@1 118 * Plugin fails when given zero-length or very short input - DONE
cannam@1 119
cannam@1 120 * Plugin fails when given "all digital zeros" input - DONE
cannam@1 121
cannam@1 122 * Plugin fails when given input that exceeds +/-1 - DONE
cannam@1 123
cannam@1 124 * Plugin fails when given "normal" random input (just in case!) - DONE
cannam@1 125
cannam@0 126 * Plugin returns different results if another instance is
cannam@3 127 constructed and run "interleaved" with it (from same thread) - DONE
cannam@0 128
cannam@0 129 * Plugin's returned timestamps do not change as expected when
cannam@0 130 run with a different base timestamp for input (though there
cannam@5 131 could be legitimate reasons for this) - DONE
cannam@0 132
cannam@0 133 * Plugin produces different results on second run, after reset
cannam@3 134 called - DONE
cannam@0 135
cannam@0 136 * Initial value of a parameter on plugin construction differs
cannam@0 137 from its default value (i.e. plugin produces different
cannam@0 138 results depending on whether parameter is set explicitly by
cannam@6 139 host to default value or not) - DONE
cannam@0 140
cannam@0 141 * If a plugin reports any programs, selecting default program
cannam@6 142 explicitly changes results (as for default parameters) - DONE
cannam@0 143
cannam@0 144 * Output feature does not hasTimestamp when output type is
cannam@5 145 VariableSampleRate - DONE
cannam@0 146
cannam@0 147 * Output feature hasTimestamp or hasDuration when output type is
cannam@5 148 OneSamplePerStep (warning only, this is not an error) - DONE
cannam@0 149
cannam@0 150 * Plugin returns features whose output numbers do not have
cannam@5 151 a corresponding record in output descriptor list - DONE
cannam@0 152
cannam@0 153 * Plugin fails to return any features on some output (warning
cannam@5 154 only) - DONE
cannam@0 155
cannam@0 156 * Constructor takes a long time to run. A fuzzy concept, but
cannam@0 157 suggests that some work should have been deferred to
cannam@6 158 initialise(). Warning only - DONE
cannam@6 159
cannam@6 160 * Plugin fails gracelessly when constructed with "weird" sample
cannam@6 161 rate or initialised with "wrong" step size, block size, or
cannam@6 162 number of channels
cannam@0 163
cannam@0 164 Well, that's quite a lot of tests already. What else?
cannam@0 165
cannam@0 166 */
cannam@0 167
cannam@0 168 bool good = true;
cannam@0 169
cannam@0 170 try {
cannam@0 171
Chris@39 172 if (m_options & Test::SingleTest) {
cannam@4 173
Chris@39 174 if (registry().find(m_singleTest) != registry().end()) {
Chris@39 175
Chris@39 176 good = performTest(m_singleTest, notes, warnings, errors);
Chris@39 177
Chris@39 178 } else {
Chris@39 179
Chris@39 180 std::cout << " ** ERROR: Unknown single-test id \""
Chris@39 181 << m_singleTest << "\"" << std::endl;
Chris@39 182 good = false;
Chris@39 183 }
Chris@39 184
Chris@39 185 } else {
Chris@39 186
Chris@39 187 for (Registry::const_iterator i = registry().begin();
Chris@39 188 i != registry().end(); ++i) {
Chris@39 189
Chris@39 190 bool thisGood = performTest(i->first, notes, warnings, errors);
Chris@39 191 if (!thisGood) good = false;
cannam@0 192 }
cannam@0 193 }
Chris@39 194
cannam@0 195 } catch (Test::FailedToLoadPlugin) {
cannam@17 196 std::cout << " ** ERROR: Failed to load plugin (key = \"" << m_key
cannam@0 197 << "\")" << std::endl;
cannam@25 198 std::cout << " ** NOTE: Vamp plugin path is: " << std::endl;
cannam@25 199 std::vector<std::string> pp = PluginHostAdapter::getPluginPath();
cannam@25 200 for (size_t i = 0; i < pp.size(); ++i) {
cannam@25 201 std::cout << " " << pp[i] << std::endl;
cannam@25 202 }
cannam@26 203 good = false;
cannam@0 204 }
cannam@0 205
cannam@0 206 return good;
cannam@0 207 }
cannam@0 208
Chris@39 209 bool
Chris@39 210 Tester::performTest(std::string id, int &notes, int &warnings, int &errors)
Chris@39 211 {
Chris@39 212 std::cout << " -- Performing test: "
Chris@39 213 << id
Chris@39 214 << " "
Chris@39 215 << nameIndex()[id]
Chris@39 216 << std::endl;
Chris@39 217
Chris@39 218 Test *test = registry()[id]->makeTest();
Chris@39 219 Test::Results results = test->test(m_key, m_options);
Chris@39 220 delete test;
Chris@39 221
Chris@39 222 set<string> printed;
Chris@39 223
Chris@39 224 bool good = true;
Chris@39 225
Chris@39 226 for (int j = 0; j < (int)results.size(); ++j) {
Chris@39 227 string message = results[j].message();
Chris@39 228 if (printed.find(message) != printed.end()) continue;
Chris@39 229 printed.insert(message);
Chris@39 230 switch (results[j].code()) {
Chris@39 231 case Test::Result::Success:
Chris@39 232 break;
Chris@39 233 case Test::Result::Note:
Chris@39 234 std::cout << " ** NOTE: " << results[j].message() << std::endl;
Chris@39 235 ++notes;
Chris@39 236 break;
Chris@39 237 case Test::Result::Warning:
Chris@39 238 std::cout << " ** WARNING: " << results[j].message() << std::endl;
Chris@39 239 ++warnings;
Chris@39 240 break;
Chris@39 241 case Test::Result::Error:
Chris@39 242 std::cout << " ** ERROR: " << results[j].message() << std::endl;
Chris@39 243 ++errors;
Chris@39 244 good = false;
Chris@39 245 break;
Chris@39 246 }
Chris@39 247 }
Chris@39 248
Chris@39 249 return good;
Chris@39 250 }
Chris@39 251