cannam@2: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
cannam@2: 
cannam@2: /*
cannam@2:     Vamp Plugin Tester
cannam@2:     Chris Cannam, cannam@all-day-breakfast.com
cannam@2:     Centre for Digital Music, Queen Mary, University of London.
cannam@2:     Copyright 2009 QMUL.
cannam@2: 
cannam@2:     This program loads a Vamp plugin and tests its susceptibility to a
cannam@2:     number of common pitfalls, including handling of extremes of input
cannam@2:     data.  If you can think of any additional useful tests that are
cannam@2:     easily added, please send them to me.
cannam@2:   
cannam@2:     Permission is hereby granted, free of charge, to any person
cannam@2:     obtaining a copy of this software and associated documentation
cannam@2:     files (the "Software"), to deal in the Software without
cannam@2:     restriction, including without limitation the rights to use, copy,
cannam@2:     modify, merge, publish, distribute, sublicense, and/or sell copies
cannam@2:     of the Software, and to permit persons to whom the Software is
cannam@2:     furnished to do so, subject to the following conditions:
cannam@2: 
cannam@2:     The above copyright notice and this permission notice shall be
cannam@2:     included in all copies or substantial portions of the Software.
cannam@2: 
cannam@2:     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
cannam@2:     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
cannam@2:     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
cannam@2:     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
cannam@2:     ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
cannam@2:     CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
cannam@2:     WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
cannam@2: 
cannam@2:     Except as contained in this notice, the names of the Centre for
cannam@2:     Digital Music; Queen Mary, University of London; and Chris Cannam
cannam@2:     shall not be used in advertising or otherwise to promote the sale,
cannam@2:     use or other dealings in this Software without prior written
cannam@2:     authorization.
cannam@2: */
cannam@2: 
cannam@2: #include "TestMultipleRuns.h"
cannam@2: 
cannam@2: #include <vamp-hostsdk/Plugin.h>
cannam@2: using namespace Vamp;
cannam@2: 
cannam@2: #include <memory>
cannam@2: using namespace std;
cannam@2: 
cannam@2: #include <cmath>
cannam@2: 
cannam@2: Tester::TestRegistrar<TestDistinctRuns>
cannam@4: TestDistinctRuns::m_registrar("D1 Consecutive runs with separate instances");
cannam@2: 
cannam@2: Tester::TestRegistrar<TestReset>
cannam@4: TestReset::m_registrar("D2 Consecutive runs with a single instance using reset");
cannam@2: 
cannam@2: Tester::TestRegistrar<TestInterleavedRuns>
cannam@4: TestInterleavedRuns::m_registrar("D3 Simultaneous interleaved runs in a single thread");
cannam@2: 
cannam@5: Tester::TestRegistrar<TestDifferentStartTimes>
cannam@5: TestDifferentStartTimes::m_registrar("D4 Consecutive runs with different start times");
cannam@5: 
cannam@3: static const size_t _step = 1000;
cannam@3: 
cannam@2: Test::Results
cannam@8: TestDistinctRuns::test(string key, Options options)
cannam@2: {
cannam@2:     Plugin::FeatureSet f[2];
cannam@2:     int rate = 44100;
cannam@2:     Results r;
cannam@3:     float **data = 0;
cannam@3:     size_t channels = 0;
cannam@3:     size_t count = 100;
cannam@2: 
cannam@2:     for (int run = 0; run < 2; ++run) {
cannam@2:         auto_ptr<Plugin> p(load(key, rate));
cannam@3:         if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
cannam@3:         if (!data) data = createTestAudio(channels, _step, count);
cannam@3:         for (size_t i = 0; i < count; ++i) {
cannam@3:             float *ptr[channels];
cannam@3:             size_t idx = i * _step;
cannam@3:             for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
cannam@2:             RealTime timestamp = RealTime::frame2RealTime(idx, rate);
cannam@3:             Plugin::FeatureSet fs = p->process(ptr, timestamp);
cannam@2:             appendFeatures(f[run], fs);
cannam@2:         }
cannam@2:         Plugin::FeatureSet fs = p->getRemainingFeatures();
cannam@2:         appendFeatures(f[run], fs);
cannam@2:     }
cannam@3:     if (data) destroyTestAudio(data, channels);
cannam@2: 
cannam@2:     if (!(f[0] == f[1])) {
cannam@8:         Result res;
cannam@8:         string message = "Consecutive runs with separate instances produce different results";
cannam@8:         if (options & NonDeterministic) res = note(message);
cannam@8:         else res = error(message);
cannam@8:         if (options & Verbose) dump(res, f[0], f[1]);
cannam@7:         r.push_back(res);
cannam@2:     } else {
cannam@2:         r.push_back(success());
cannam@2:     }
cannam@2: 
cannam@2:     return r;
cannam@2: }
cannam@2: 
cannam@2: Test::Results
cannam@8: TestReset::test(string key, Options options)
cannam@2: {
cannam@2:     Plugin::FeatureSet f[2];
cannam@2:     int rate = 44100;
cannam@2:     Results r;
cannam@3:     float **data = 0;
cannam@3:     size_t channels = 0;
cannam@3:     size_t count = 100;
cannam@2: 
cannam@2:     auto_ptr<Plugin> p(load(key, rate));
cannam@3: 
cannam@2:     for (int run = 0; run < 2; ++run) {
cannam@2:         if (run == 1) p->reset();
cannam@11:         else if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
cannam@3:         if (!data) data = createTestAudio(channels, _step, count);
cannam@3:         for (size_t i = 0; i < count; ++i) {
cannam@3:             float *ptr[channels];
cannam@3:             size_t idx = i * _step;
cannam@3:             for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
cannam@2:             RealTime timestamp = RealTime::frame2RealTime(idx, rate);
cannam@3:             Plugin::FeatureSet fs = p->process(ptr, timestamp);
cannam@2:             appendFeatures(f[run], fs);
cannam@2:         }
cannam@2:         Plugin::FeatureSet fs = p->getRemainingFeatures();
cannam@2:         appendFeatures(f[run], fs);
cannam@2:     }
cannam@3:     if (data) destroyTestAudio(data, channels);
cannam@2: 
cannam@2:     if (!(f[0] == f[1])) {
cannam@8:         string message = "Consecutive runs with the same instance (using reset) produce different results";
cannam@8:         Result res;
cannam@8:         if (options & NonDeterministic) res = note(message);
cannam@8:         else res = error(message);
cannam@8:         if (options & Verbose) dump(res, f[0], f[1]);
cannam@3:         r.push_back(res);
cannam@2:     } else {
cannam@2:         r.push_back(success());
cannam@2:     }
cannam@2: 
cannam@2:     return r;
cannam@2: }
cannam@2: 
cannam@2: Test::Results
cannam@8: TestInterleavedRuns::test(string key, Options options)
cannam@2: {
cannam@2:     Plugin::FeatureSet f[2];
cannam@2:     int rate = 44100;
cannam@2:     Results r;
cannam@3:     float **data = 0;
cannam@3:     size_t channels = 0;
cannam@3:     size_t count = 100;
cannam@3: 
cannam@2:     Plugin *p[2];
cannam@2:     for (int run = 0; run < 2; ++run) {
cannam@2:         p[run] = load(key, rate);
cannam@3:         if (!initAdapted(p[run], channels, _step, _step, r)) {
cannam@2:             delete p[run];
cannam@2:             if (run > 0) delete p[0];
cannam@2:             return r;
cannam@2:         }
cannam@3:         if (!data) data = createTestAudio(channels, _step, count);
cannam@2:     }
cannam@3:     for (size_t i = 0; i < count; ++i) {
cannam@3:         float *ptr[channels];
cannam@3:         size_t idx = i * _step;
cannam@3:         for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
cannam@2:         RealTime timestamp = RealTime::frame2RealTime(idx, rate);
cannam@2:         for (int run = 0; run < 2; ++run) {
cannam@3:             Plugin::FeatureSet fs = p[run]->process(ptr, timestamp);
cannam@2:             appendFeatures(f[run], fs);
cannam@2:         }
cannam@2:     }
cannam@2:     for (int run = 0; run < 2; ++run) {
cannam@2:         Plugin::FeatureSet fs = p[run]->getRemainingFeatures();
cannam@2:         appendFeatures(f[run], fs);
cannam@2:         delete p[run];
cannam@2:     }
cannam@2: 
cannam@3:     if (data) destroyTestAudio(data, channels);
cannam@2: 
cannam@2:     if (!(f[0] == f[1])) {
cannam@8:         string message = "Simultaneous runs with separate instances produce different results";
cannam@8:         Result res;
cannam@8:         if (options & NonDeterministic) res = note(message);
cannam@8:         else res = error(message);
cannam@8:         if (options & Verbose) dump(res, f[0], f[1]);
cannam@7:         r.push_back(res);
cannam@2:     } else {
cannam@2:         r.push_back(success());
cannam@2:     }
cannam@2: 
cannam@2:     return r;
cannam@2: }
cannam@5: 
cannam@5: Test::Results
cannam@8: TestDifferentStartTimes::test(string key, Options options)
cannam@5: {
cannam@5:     Plugin::FeatureSet f[2];
cannam@5:     int rate = 44100;
cannam@5:     Results r;
cannam@5:     float **data = 0;
cannam@5:     size_t channels = 0;
cannam@5:     size_t count = 100;
cannam@5: 
cannam@5:     for (int run = 0; run < 2; ++run) {
cannam@5:         auto_ptr<Plugin> p(load(key, rate));
cannam@5:         if (!initAdapted(p.get(), channels, _step, _step, r)) return r;
cannam@5:         if (!data) data = createTestAudio(channels, _step, count);
cannam@5:         for (size_t i = 0; i < count; ++i) {
cannam@5:             float *ptr[channels];
cannam@5:             size_t idx = i * _step;
cannam@5:             for (size_t c = 0; c < channels; ++c) ptr[c] = data[c] + idx;
cannam@5:             RealTime timestamp = RealTime::frame2RealTime(idx, rate);
cannam@5:             if (run == 1) timestamp = timestamp + RealTime::fromSeconds(10);
cannam@5:             Plugin::FeatureSet fs = p->process(ptr, timestamp);
cannam@5:             appendFeatures(f[run], fs);
cannam@5:         }
cannam@5:         Plugin::FeatureSet fs = p->getRemainingFeatures();
cannam@5:         appendFeatures(f[run], fs);
cannam@5:     }
cannam@5:     if (data) destroyTestAudio(data, channels);
cannam@5: 
cannam@5:     if (f[0] == f[1]) {
cannam@8:         string message = "Consecutive runs with different starting timestamps produce the same result";
cannam@8:         Result res;
cannam@8:         if (options & NonDeterministic) res = note(message);
cannam@13:         else res = warning(message);
cannam@8:         if (options & Verbose) dump(res, f[0], f[1]);
cannam@7:         r.push_back(res);
cannam@5:     } else {
cannam@5:         r.push_back(success());
cannam@5:     }
cannam@5: 
cannam@5:     return r;
cannam@5: }