cannam@2
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
cannam@2
|
2
|
cannam@2
|
3 /*
|
cannam@2
|
4 Vamp Plugin Tester
|
cannam@2
|
5 Chris Cannam, cannam@all-day-breakfast.com
|
cannam@2
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@42
|
7 Copyright 2009-2014 QMUL.
|
cannam@2
|
8
|
cannam@2
|
9 This program loads a Vamp plugin and tests its susceptibility to a
|
cannam@2
|
10 number of common pitfalls, including handling of extremes of input
|
cannam@2
|
11 data. If you can think of any additional useful tests that are
|
cannam@2
|
12 easily added, please send them to me.
|
cannam@2
|
13
|
cannam@2
|
14 Permission is hereby granted, free of charge, to any person
|
cannam@2
|
15 obtaining a copy of this software and associated documentation
|
cannam@2
|
16 files (the "Software"), to deal in the Software without
|
cannam@2
|
17 restriction, including without limitation the rights to use, copy,
|
cannam@2
|
18 modify, merge, publish, distribute, sublicense, and/or sell copies
|
cannam@2
|
19 of the Software, and to permit persons to whom the Software is
|
cannam@2
|
20 furnished to do so, subject to the following conditions:
|
cannam@2
|
21
|
cannam@2
|
22 The above copyright notice and this permission notice shall be
|
cannam@2
|
23 included in all copies or substantial portions of the Software.
|
cannam@2
|
24
|
cannam@2
|
25 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
cannam@2
|
26 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
cannam@2
|
27 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
cannam@2
|
28 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
|
cannam@2
|
29 ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
|
cannam@2
|
30 CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
cannam@2
|
31 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
cannam@2
|
32
|
cannam@2
|
33 Except as contained in this notice, the names of the Centre for
|
cannam@2
|
34 Digital Music; Queen Mary, University of London; and Chris Cannam
|
cannam@2
|
35 shall not be used in advertising or otherwise to promote the sale,
|
cannam@2
|
36 use or other dealings in this Software without prior written
|
cannam@2
|
37 authorization.
|
cannam@2
|
38 */
|
cannam@2
|
39
|
cannam@2
|
40 #include "TestMultipleRuns.h"
|
cannam@2
|
41
|
cannam@2
|
42 #include <vamp-hostsdk/Plugin.h>
|
cannam@2
|
43 using namespace Vamp;
|
cannam@2
|
44
|
cannam@2
|
45 #include <memory>
|
cannam@2
|
46 using namespace std;
|
cannam@2
|
47
|
cannam@2
|
48 #include <cmath>
|
cannam@2
|
49
|
cannam@2
|
50 Tester::TestRegistrar<TestDistinctRuns>
|
Chris@39
|
51 TestDistinctRuns::m_registrar("D1", "Consecutive runs with separate instances");
|
cannam@2
|
52
|
cannam@2
|
53 Tester::TestRegistrar<TestReset>
|
Chris@39
|
54 TestReset::m_registrar("D2", "Consecutive runs with a single instance using reset");
|
cannam@2
|
55
|
cannam@2
|
56 Tester::TestRegistrar<TestInterleavedRuns>
|
Chris@39
|
57 TestInterleavedRuns::m_registrar("D3", "Simultaneous interleaved runs in a single thread");
|
cannam@2
|
58
|
cannam@5
|
59 Tester::TestRegistrar<TestDifferentStartTimes>
|
Chris@39
|
60 TestDifferentStartTimes::m_registrar("D4", "Consecutive runs with different start times");
|
cannam@5
|
61
|
cannam@3
|
62 static const size_t _step = 1000;
|
cannam@3
|
63
|
cannam@2
|
64 Test::Results
|
cannam@8
|
65 TestDistinctRuns::test(string key, Options options)
|
cannam@2
|
66 {
|
cannam@2
|
67 Plugin::FeatureSet f[2];
|
cannam@2
|
68 int rate = 44100;
|
cannam@2
|
69 Results r;
|
cannam@3
|
70 float **data = 0;
|
cannam@3
|
71 size_t channels = 0;
|
cannam@3
|
72 size_t count = 100;
|
cannam@2
|
73
|
cannam@2
|
74 for (int run = 0; run < 2; ++run) {
|
cannam@2
|
75 auto_ptr<Plugin> p(load(key, rate));
|
cannam@3
|
76 if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
|
cannam@3
|
77 if (!data) data = createTestAudio(channels, _step, count);
|
cannam@3
|
78 for (size_t i = 0; i < count; ++i) {
|
Chris@67
|
79 float **ptr = new float *[channels];
|
cannam@3
|
80 size_t idx = i * _step;
|
cannam@3
|
81 for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
|
cannam@2
|
82 RealTime timestamp = RealTime::frame2RealTime(idx, rate);
|
cannam@3
|
83 Plugin::FeatureSet fs = p->process(ptr, timestamp);
|
Chris@67
|
84 delete[] ptr;
|
cannam@2
|
85 appendFeatures(f[run], fs);
|
cannam@2
|
86 }
|
cannam@2
|
87 Plugin::FeatureSet fs = p->getRemainingFeatures();
|
cannam@2
|
88 appendFeatures(f[run], fs);
|
cannam@2
|
89 }
|
cannam@3
|
90 if (data) destroyTestAudio(data, channels);
|
cannam@2
|
91
|
cannam@2
|
92 if (!(f[0] == f[1])) {
|
cannam@8
|
93 Result res;
|
cannam@8
|
94 string message = "Consecutive runs with separate instances produce different results";
|
cannam@8
|
95 if (options & NonDeterministic) res = note(message);
|
cannam@8
|
96 else res = error(message);
|
Chris@52
|
97 if (options & Verbose) dumpDiff(res, f[0], f[1]);
|
cannam@7
|
98 r.push_back(res);
|
cannam@2
|
99 } else {
|
cannam@2
|
100 r.push_back(success());
|
cannam@2
|
101 }
|
cannam@2
|
102
|
cannam@2
|
103 return r;
|
cannam@2
|
104 }
|
cannam@2
|
105
|
cannam@2
|
106 Test::Results
|
cannam@8
|
107 TestReset::test(string key, Options options)
|
cannam@2
|
108 {
|
cannam@2
|
109 Plugin::FeatureSet f[2];
|
cannam@2
|
110 int rate = 44100;
|
cannam@2
|
111 Results r;
|
cannam@3
|
112 float **data = 0;
|
cannam@3
|
113 size_t channels = 0;
|
cannam@3
|
114 size_t count = 100;
|
cannam@2
|
115
|
cannam@2
|
116 auto_ptr<Plugin> p(load(key, rate));
|
cannam@3
|
117
|
cannam@2
|
118 for (int run = 0; run < 2; ++run) {
|
cannam@2
|
119 if (run == 1) p->reset();
|
cannam@11
|
120 else if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
|
cannam@3
|
121 if (!data) data = createTestAudio(channels, _step, count);
|
cannam@3
|
122 for (size_t i = 0; i < count; ++i) {
|
Chris@67
|
123 float **ptr = new float *[channels];
|
cannam@3
|
124 size_t idx = i * _step;
|
cannam@3
|
125 for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
|
cannam@2
|
126 RealTime timestamp = RealTime::frame2RealTime(idx, rate);
|
cannam@3
|
127 Plugin::FeatureSet fs = p->process(ptr, timestamp);
|
Chris@67
|
128 delete[] ptr;
|
cannam@2
|
129 appendFeatures(f[run], fs);
|
cannam@2
|
130 }
|
cannam@2
|
131 Plugin::FeatureSet fs = p->getRemainingFeatures();
|
cannam@2
|
132 appendFeatures(f[run], fs);
|
cannam@2
|
133 }
|
cannam@3
|
134 if (data) destroyTestAudio(data, channels);
|
cannam@2
|
135
|
cannam@2
|
136 if (!(f[0] == f[1])) {
|
cannam@8
|
137 string message = "Consecutive runs with the same instance (using reset) produce different results";
|
cannam@8
|
138 Result res;
|
cannam@8
|
139 if (options & NonDeterministic) res = note(message);
|
cannam@8
|
140 else res = error(message);
|
Chris@52
|
141 if (options & Verbose) dumpDiff(res, f[0], f[1]);
|
cannam@3
|
142 r.push_back(res);
|
cannam@2
|
143 } else {
|
cannam@2
|
144 r.push_back(success());
|
cannam@2
|
145 }
|
cannam@2
|
146
|
cannam@2
|
147 return r;
|
cannam@2
|
148 }
|
cannam@2
|
149
|
cannam@2
|
150 Test::Results
|
cannam@8
|
151 TestInterleavedRuns::test(string key, Options options)
|
cannam@2
|
152 {
|
cannam@2
|
153 Plugin::FeatureSet f[2];
|
cannam@2
|
154 int rate = 44100;
|
cannam@2
|
155 Results r;
|
cannam@3
|
156 float **data = 0;
|
cannam@3
|
157 size_t channels = 0;
|
cannam@3
|
158 size_t count = 100;
|
cannam@3
|
159
|
cannam@2
|
160 Plugin *p[2];
|
cannam@2
|
161 for (int run = 0; run < 2; ++run) {
|
cannam@2
|
162 p[run] = load(key, rate);
|
cannam@3
|
163 if (!initAdapted(p[run], channels, _step, _step, r)) {
|
cannam@2
|
164 delete p[run];
|
cannam@2
|
165 if (run > 0) delete p[0];
|
cannam@2
|
166 return r;
|
cannam@2
|
167 }
|
cannam@3
|
168 if (!data) data = createTestAudio(channels, _step, count);
|
cannam@2
|
169 }
|
cannam@3
|
170 for (size_t i = 0; i < count; ++i) {
|
Chris@67
|
171 float **ptr = new float *[channels];
|
cannam@3
|
172 size_t idx = i * _step;
|
cannam@3
|
173 for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
|
cannam@2
|
174 RealTime timestamp = RealTime::frame2RealTime(idx, rate);
|
cannam@2
|
175 for (int run = 0; run < 2; ++run) {
|
cannam@3
|
176 Plugin::FeatureSet fs = p[run]->process(ptr, timestamp);
|
cannam@2
|
177 appendFeatures(f[run], fs);
|
cannam@2
|
178 }
|
Chris@67
|
179 delete[] ptr;
|
cannam@2
|
180 }
|
cannam@2
|
181 for (int run = 0; run < 2; ++run) {
|
cannam@2
|
182 Plugin::FeatureSet fs = p[run]->getRemainingFeatures();
|
cannam@2
|
183 appendFeatures(f[run], fs);
|
cannam@2
|
184 delete p[run];
|
cannam@2
|
185 }
|
cannam@2
|
186
|
cannam@3
|
187 if (data) destroyTestAudio(data, channels);
|
cannam@2
|
188
|
cannam@2
|
189 if (!(f[0] == f[1])) {
|
cannam@8
|
190 string message = "Simultaneous runs with separate instances produce different results";
|
cannam@8
|
191 Result res;
|
cannam@8
|
192 if (options & NonDeterministic) res = note(message);
|
cannam@8
|
193 else res = error(message);
|
Chris@52
|
194 if (options & Verbose) dumpDiff(res, f[0], f[1]);
|
cannam@7
|
195 r.push_back(res);
|
cannam@2
|
196 } else {
|
cannam@2
|
197 r.push_back(success());
|
cannam@2
|
198 }
|
cannam@2
|
199
|
cannam@2
|
200 return r;
|
cannam@2
|
201 }
|
cannam@5
|
202
|
cannam@5
|
203 Test::Results
|
cannam@8
|
204 TestDifferentStartTimes::test(string key, Options options)
|
cannam@5
|
205 {
|
cannam@5
|
206 Plugin::FeatureSet f[2];
|
cannam@5
|
207 int rate = 44100;
|
cannam@5
|
208 Results r;
|
cannam@5
|
209 float **data = 0;
|
cannam@5
|
210 size_t channels = 0;
|
cannam@5
|
211 size_t count = 100;
|
cannam@5
|
212
|
cannam@5
|
213 for (int run = 0; run < 2; ++run) {
|
cannam@5
|
214 auto_ptr<Plugin> p(load(key, rate));
|
cannam@5
|
215 if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
|
cannam@5
|
216 if (!data) data = createTestAudio(channels, _step, count);
|
cannam@5
|
217 for (size_t i = 0; i < count; ++i) {
|
Chris@67
|
218 float **ptr = new float *[channels];
|
cannam@5
|
219 size_t idx = i * _step;
|
cannam@5
|
220 for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
|
cannam@5
|
221 RealTime timestamp = RealTime::frame2RealTime(idx, rate);
|
cannam@5
|
222 if (run == 1) timestamp = timestamp + RealTime::fromSeconds(10);
|
cannam@5
|
223 Plugin::FeatureSet fs = p->process(ptr, timestamp);
|
Chris@67
|
224 delete[] ptr;
|
cannam@5
|
225 appendFeatures(f[run], fs);
|
cannam@5
|
226 }
|
cannam@5
|
227 Plugin::FeatureSet fs = p->getRemainingFeatures();
|
cannam@5
|
228 appendFeatures(f[run], fs);
|
cannam@5
|
229 }
|
cannam@5
|
230 if (data) destroyTestAudio(data, channels);
|
cannam@5
|
231
|
cannam@5
|
232 if (f[0] == f[1]) {
|
cannam@8
|
233 Result res;
|
Chris@53
|
234 if (containsTimestamps(f[0])) {
|
Chris@53
|
235 string message = "Consecutive runs with different starting timestamps produce the same result";
|
Chris@53
|
236 if (options & NonDeterministic) {
|
Chris@53
|
237 res = note(message);
|
Chris@53
|
238 } else {
|
Chris@53
|
239 res = warning(message);
|
Chris@53
|
240 }
|
Chris@53
|
241 if (options & Verbose) {
|
Chris@53
|
242 cout << res.message() << endl;
|
Chris@53
|
243 dump(f[0], false);
|
Chris@53
|
244 }
|
Chris@53
|
245 } else {
|
Chris@53
|
246 res = note("Consecutive runs with different starting timestamps produce the same result (but result features contain no timestamps, so this is probably all right)");
|
Chris@52
|
247 }
|
cannam@7
|
248 r.push_back(res);
|
cannam@5
|
249 } else {
|
cannam@5
|
250 r.push_back(success());
|
cannam@5
|
251 }
|
cannam@5
|
252
|
cannam@5
|
253 return r;
|
cannam@5
|
254 }
|