cannam@0
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
cannam@0
|
2
|
cannam@0
|
3 /*
|
cannam@0
|
4 Vamp Plugin Tester
|
cannam@0
|
5 Chris Cannam, cannam@all-day-breakfast.com
|
cannam@0
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@42
|
7 Copyright 2009-2014 QMUL.
|
cannam@0
|
8
|
cannam@0
|
9 This program loads a Vamp plugin and tests its susceptibility to a
|
cannam@0
|
10 number of common pitfalls, including handling of extremes of input
|
cannam@0
|
11 data. If you can think of any additional useful tests that are
|
cannam@0
|
12 easily added, please send them to me.
|
cannam@0
|
13
|
cannam@0
|
14 Permission is hereby granted, free of charge, to any person
|
cannam@0
|
15 obtaining a copy of this software and associated documentation
|
cannam@0
|
16 files (the "Software"), to deal in the Software without
|
cannam@0
|
17 restriction, including without limitation the rights to use, copy,
|
cannam@0
|
18 modify, merge, publish, distribute, sublicense, and/or sell copies
|
cannam@0
|
19 of the Software, and to permit persons to whom the Software is
|
cannam@0
|
20 furnished to do so, subject to the following conditions:
|
cannam@0
|
21
|
cannam@0
|
22 The above copyright notice and this permission notice shall be
|
cannam@0
|
23 included in all copies or substantial portions of the Software.
|
cannam@0
|
24
|
cannam@0
|
25 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
cannam@0
|
26 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
cannam@0
|
27 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
cannam@0
|
28 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
|
cannam@0
|
29 ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
|
cannam@0
|
30 CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
cannam@0
|
31 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
cannam@0
|
32
|
cannam@0
|
33 Except as contained in this notice, the names of the Centre for
|
cannam@0
|
34 Digital Music; Queen Mary, University of London; and Chris Cannam
|
cannam@0
|
35 shall not be used in advertising or otherwise to promote the sale,
|
cannam@0
|
36 use or other dealings in this Software without prior written
|
cannam@0
|
37 authorization.
|
cannam@0
|
38 */
|
cannam@0
|
39
|
cannam@0
|
40 #include <vamp-hostsdk/PluginHostAdapter.h>
|
cannam@0
|
41 #include <vamp-hostsdk/PluginInputDomainAdapter.h>
|
cannam@0
|
42 #include <vamp-hostsdk/PluginLoader.h>
|
cannam@0
|
43
|
cannam@0
|
44 #include <iostream>
|
cannam@0
|
45
|
cannam@0
|
46 #include <cstring>
|
cannam@0
|
47 #include <cstdlib>
|
cannam@0
|
48 #include <cmath>
|
cannam@4
|
49 #include <set>
|
cannam@0
|
50
|
cannam@0
|
51 #include "Tester.h"
|
cannam@0
|
52
|
cannam@0
|
53 using Vamp::Plugin;
|
cannam@0
|
54 using Vamp::PluginHostAdapter;
|
cannam@0
|
55 using Vamp::RealTime;
|
cannam@0
|
56 using Vamp::HostExt::PluginLoader;
|
cannam@0
|
57 using Vamp::HostExt::PluginWrapper;
|
cannam@0
|
58 using Vamp::HostExt::PluginInputDomainAdapter;
|
cannam@0
|
59
|
cannam@0
|
60 using namespace std;
|
cannam@0
|
61
|
Chris@39
|
62 Tester::Tester(std::string key, Test::Options options, std::string singleTestId) :
|
cannam@8
|
63 m_key(key),
|
Chris@39
|
64 m_options(options),
|
Chris@39
|
65 m_singleTest(singleTestId)
|
cannam@0
|
66 {
|
cannam@0
|
67 }
|
cannam@0
|
68
|
cannam@0
|
69 Tester::~Tester()
|
cannam@0
|
70 {
|
cannam@0
|
71 }
|
cannam@0
|
72
|
Chris@39
|
73 Tester::NameIndex &
|
Chris@39
|
74 Tester::nameIndex()
|
Chris@39
|
75 {
|
Chris@39
|
76 static NameIndex ix;
|
Chris@39
|
77 return ix;
|
Chris@39
|
78 }
|
Chris@39
|
79
|
cannam@0
|
80 Tester::Registry &
|
cannam@0
|
81 Tester::registry()
|
cannam@0
|
82 {
|
cannam@0
|
83 static Registry r;
|
cannam@0
|
84 return r;
|
cannam@0
|
85 }
|
cannam@0
|
86
|
Chris@40
|
87 void
|
Chris@40
|
88 Tester::listTests()
|
Chris@40
|
89 {
|
Chris@40
|
90 cout << endl;
|
Chris@40
|
91 cout << "Total tests: " << nameIndex().size() << "\n" << endl;
|
Chris@40
|
92 cout << "ID | Name" << endl;
|
Chris@40
|
93 cout << "---+-----" << endl;
|
Chris@40
|
94 for (NameIndex::const_iterator i = nameIndex().begin();
|
Chris@40
|
95 i != nameIndex().end(); ++i) {
|
Chris@40
|
96 cout << i->first << " | " << i->second << endl;
|
Chris@40
|
97 }
|
Chris@40
|
98 cout << endl;
|
Chris@40
|
99 }
|
Chris@40
|
100
|
cannam@0
|
101 bool
|
cannam@4
|
102 Tester::test(int ¬es, int &warnings, int &errors)
|
cannam@0
|
103 {
|
cannam@0
|
104 /*
|
cannam@0
|
105
|
cannam@0
|
106 Things I would like to see tested:
|
cannam@0
|
107
|
cannam@0
|
108 * Identifiers for parameters, outputs, or plugin itself contain
|
cannam@0
|
109 illegal characters - DONE
|
cannam@0
|
110
|
cannam@0
|
111 * Any of the plugin's name, maker etc fields are empty - DONE
|
cannam@0
|
112
|
cannam@1
|
113 * Default value of a parameter is not quantized as specified - DONE
|
cannam@1
|
114
|
cannam@1
|
115 * Parameter minValue >= maxValue, or defaultValue < minValue
|
cannam@1
|
116 or > maxValue - DONE
|
cannam@1
|
117
|
cannam@1
|
118 * Plugin fails when given zero-length or very short input - DONE
|
cannam@1
|
119
|
cannam@1
|
120 * Plugin fails when given "all digital zeros" input - DONE
|
cannam@1
|
121
|
cannam@1
|
122 * Plugin fails when given input that exceeds +/-1 - DONE
|
cannam@1
|
123
|
cannam@1
|
124 * Plugin fails when given "normal" random input (just in case!) - DONE
|
cannam@1
|
125
|
cannam@0
|
126 * Plugin returns different results if another instance is
|
cannam@3
|
127 constructed and run "interleaved" with it (from same thread) - DONE
|
cannam@0
|
128
|
cannam@0
|
129 * Plugin's returned timestamps do not change as expected when
|
cannam@0
|
130 run with a different base timestamp for input (though there
|
cannam@5
|
131 could be legitimate reasons for this) - DONE
|
cannam@0
|
132
|
cannam@0
|
133 * Plugin produces different results on second run, after reset
|
cannam@3
|
134 called - DONE
|
cannam@0
|
135
|
cannam@0
|
136 * Initial value of a parameter on plugin construction differs
|
cannam@0
|
137 from its default value (i.e. plugin produces different
|
cannam@0
|
138 results depending on whether parameter is set explicitly by
|
cannam@6
|
139 host to default value or not) - DONE
|
cannam@0
|
140
|
cannam@0
|
141 * If a plugin reports any programs, selecting default program
|
cannam@6
|
142 explicitly changes results (as for default parameters) - DONE
|
cannam@0
|
143
|
cannam@0
|
144 * Output feature does not hasTimestamp when output type is
|
cannam@5
|
145 VariableSampleRate - DONE
|
cannam@0
|
146
|
cannam@0
|
147 * Output feature hasTimestamp or hasDuration when output type is
|
cannam@5
|
148 OneSamplePerStep (warning only, this is not an error) - DONE
|
cannam@0
|
149
|
cannam@0
|
150 * Plugin returns features whose output numbers do not have
|
cannam@5
|
151 a corresponding record in output descriptor list - DONE
|
cannam@0
|
152
|
cannam@0
|
153 * Plugin fails to return any features on some output (warning
|
cannam@5
|
154 only) - DONE
|
cannam@0
|
155
|
cannam@0
|
156 * Constructor takes a long time to run. A fuzzy concept, but
|
cannam@0
|
157 suggests that some work should have been deferred to
|
cannam@6
|
158 initialise(). Warning only - DONE
|
cannam@6
|
159
|
cannam@6
|
160 * Plugin fails gracelessly when constructed with "weird" sample
|
cannam@6
|
161 rate or initialised with "wrong" step size, block size, or
|
cannam@6
|
162 number of channels
|
cannam@0
|
163
|
cannam@0
|
164 Well, that's quite a lot of tests already. What else?
|
cannam@0
|
165
|
cannam@0
|
166 */
|
cannam@0
|
167
|
cannam@0
|
168 bool good = true;
|
cannam@0
|
169
|
cannam@0
|
170 try {
|
cannam@0
|
171
|
Chris@39
|
172 if (m_options & Test::SingleTest) {
|
cannam@4
|
173
|
Chris@39
|
174 if (registry().find(m_singleTest) != registry().end()) {
|
Chris@39
|
175
|
Chris@39
|
176 good = performTest(m_singleTest, notes, warnings, errors);
|
Chris@39
|
177
|
Chris@39
|
178 } else {
|
Chris@39
|
179
|
Chris@39
|
180 std::cout << " ** ERROR: Unknown single-test id \""
|
Chris@39
|
181 << m_singleTest << "\"" << std::endl;
|
Chris@39
|
182 good = false;
|
Chris@39
|
183 }
|
Chris@39
|
184
|
Chris@39
|
185 } else {
|
Chris@39
|
186
|
Chris@39
|
187 for (Registry::const_iterator i = registry().begin();
|
Chris@39
|
188 i != registry().end(); ++i) {
|
Chris@39
|
189
|
Chris@39
|
190 bool thisGood = performTest(i->first, notes, warnings, errors);
|
Chris@39
|
191 if (!thisGood) good = false;
|
cannam@0
|
192 }
|
cannam@0
|
193 }
|
Chris@39
|
194
|
cannam@0
|
195 } catch (Test::FailedToLoadPlugin) {
|
cannam@17
|
196 std::cout << " ** ERROR: Failed to load plugin (key = \"" << m_key
|
cannam@0
|
197 << "\")" << std::endl;
|
cannam@25
|
198 std::cout << " ** NOTE: Vamp plugin path is: " << std::endl;
|
cannam@25
|
199 std::vector<std::string> pp = PluginHostAdapter::getPluginPath();
|
cannam@25
|
200 for (size_t i = 0; i < pp.size(); ++i) {
|
cannam@25
|
201 std::cout << " " << pp[i] << std::endl;
|
cannam@25
|
202 }
|
cannam@26
|
203 good = false;
|
cannam@0
|
204 }
|
cannam@0
|
205
|
cannam@0
|
206 return good;
|
cannam@0
|
207 }
|
cannam@0
|
208
|
Chris@39
|
209 bool
|
Chris@39
|
210 Tester::performTest(std::string id, int ¬es, int &warnings, int &errors)
|
Chris@39
|
211 {
|
Chris@39
|
212 std::cout << " -- Performing test: "
|
Chris@39
|
213 << id
|
Chris@39
|
214 << " "
|
Chris@39
|
215 << nameIndex()[id]
|
Chris@39
|
216 << std::endl;
|
Chris@39
|
217
|
Chris@39
|
218 Test *test = registry()[id]->makeTest();
|
Chris@39
|
219 Test::Results results = test->test(m_key, m_options);
|
Chris@39
|
220 delete test;
|
Chris@39
|
221
|
Chris@39
|
222 set<string> printed;
|
Chris@39
|
223
|
Chris@39
|
224 bool good = true;
|
Chris@39
|
225
|
Chris@39
|
226 for (int j = 0; j < (int)results.size(); ++j) {
|
Chris@39
|
227 string message = results[j].message();
|
Chris@39
|
228 if (printed.find(message) != printed.end()) continue;
|
Chris@39
|
229 printed.insert(message);
|
Chris@39
|
230 switch (results[j].code()) {
|
Chris@39
|
231 case Test::Result::Success:
|
Chris@39
|
232 break;
|
Chris@39
|
233 case Test::Result::Note:
|
Chris@39
|
234 std::cout << " ** NOTE: " << results[j].message() << std::endl;
|
Chris@39
|
235 ++notes;
|
Chris@39
|
236 break;
|
Chris@39
|
237 case Test::Result::Warning:
|
Chris@39
|
238 std::cout << " ** WARNING: " << results[j].message() << std::endl;
|
Chris@39
|
239 ++warnings;
|
Chris@39
|
240 break;
|
Chris@39
|
241 case Test::Result::Error:
|
Chris@39
|
242 std::cout << " ** ERROR: " << results[j].message() << std::endl;
|
Chris@39
|
243 ++errors;
|
Chris@39
|
244 good = false;
|
Chris@39
|
245 break;
|
Chris@39
|
246 }
|
Chris@39
|
247 }
|
Chris@39
|
248
|
Chris@39
|
249 return good;
|
Chris@39
|
250 }
|
Chris@39
|
251
|