Revision 8:3019cb6b538d

View differences:

Test.cpp
169 169
    for (Plugin::FeatureSet::const_iterator fsi = fs.begin();
170 170
         fsi != fs.end(); ++fsi) {
171 171
        int output = fsi->first;
172
        std::cerr << "Output " << output << ":" << std::endl;
172
        std::cout << "Output " << output << ":" << std::endl;
173 173
        const Plugin::FeatureList &fl = fsi->second;
174 174
        for (int i = 0; i < (int)fl.size(); ++i) {
175
            std::cerr << "  Feature " << i << ":" << std::endl;
175
            std::cout << "  Feature " << i << ":" << std::endl;
176 176
            const Plugin::Feature &f = fl[i];
177
            std::cerr << "    Timestamp: " << (f.hasTimestamp ? "(none)" : f.timestamp.toText()) << std::endl;
178
            std::cerr << "    Duration: " << (f.hasDuration ? "(none)" : f.duration.toText()) << std::endl;
179
            std::cerr << "    Label: " << (f.label == "" ? "(none)" : f.label) << std::endl;
180
            std::cerr << "    Value: " << (f.values.empty() ? "(none)" : "");
177
            std::cout << "    Timestamp: " << (f.hasTimestamp ? "(none)" : f.timestamp.toText()) << std::endl;
178
            std::cout << "    Duration: " << (f.hasDuration ? "(none)" : f.duration.toText()) << std::endl;
179
            std::cout << "    Label: " << (f.label == "" ? "(none)" : f.label) << std::endl;
180
            std::cout << "    Value: " << (f.values.empty() ? "(none)" : "");
181 181
            for (int j = 0; j < (int)f.values.size(); ++j) {
182
                std::cerr << f.values[j] << " ";
182
                std::cout << f.values[j] << " ";
183 183
            }
184
            std::cerr << std::endl;
184
            std::cout << std::endl;
185 185
        }
186 186
    }
187 187
}
......
191 191
           const Plugin::FeatureSet &a,
192 192
           const Plugin::FeatureSet &b)
193 193
{
194
    std::cerr << r.message() << std::endl;
195
    std::cerr << "\nFirst result set:" << std::endl;
194
    std::cout << r.message() << std::endl;
195
    std::cout << "\nFirst result set:" << std::endl;
196 196
    dump(a);
197
    std::cerr << "\nSecond result set:" << std::endl;
197
    std::cout << "\nSecond result set:" << std::endl;
198 198
    dump(b);
199
    std::cerr << std::endl;
199
    std::cout << std::endl;
200 200
}
201 201

  
202 202
bool
Test.h
48 48
{
49 49
public:
50 50
    virtual ~Test();
51

  
52
    enum Option {
53
        NoOption = 0x0,
54
        NonDeterministic = 0x1,
55
        Verbose = 0x2
56
    };
57
    typedef int Options;
51 58
    
52 59
    class Result {
53 60

  
54 61
    public:
55 62
        enum Code { Success, Note, Warning, Error };
56 63

  
64
        Result() : m_code(Success) { }
57 65
        Result(Code c, std::string m) : m_code(c), m_message(m) { }
58 66

  
59 67
        Code code() const { return m_code; }
60 68
        std::string message() const { return m_message; }
61

  
69
        
62 70
    protected:
63 71
        Code m_code;
64 72
        std::string m_message;
......
74 82
    class FailedToLoadPlugin { };
75 83

  
76 84
    // may throw FailedToLoadPlugin
77
    virtual Results test(std::string key) = 0;
85
    virtual Results test(std::string key, Options) = 0;
78 86

  
79 87
protected:
80 88
    Test();
TestDefaults.cpp
60 60
static const size_t _step = 1000;
61 61

  
62 62
Test::Results
63
TestDefaultProgram::test(string key)
63
TestDefaultProgram::test(string key, Options options)
64 64
{
65 65
    Plugin::FeatureSet f[2];
66 66
    int rate = 44100;
......
91 91
    if (data) destroyTestAudio(data, channels);
92 92

  
93 93
    if (!(f[0] == f[1])) {
94
        Result res = warning("Explicitly setting current program to its supposed current value changes the results");
95
        dump(res, f[0], f[1]);
94
        string message = "Explicitly setting current program to its supposed current value changes the results";
95
        Result res;
96
        if (options & NonDeterministic) res = note(message);
97
        else res = error(message);
98
        if (options & Verbose) dump(res, f[0], f[1]);
96 99
        r.push_back(res);
97 100
    } else {
98 101
        r.push_back(success());
......
102 105
}
103 106

  
104 107
Test::Results
105
TestDefaultParameters::test(string key)
108
TestDefaultParameters::test(string key, Options options)
106 109
{
107 110
    Plugin::FeatureSet f[2];
108 111
    int rate = 44100;
......
139 142
    if (data) destroyTestAudio(data, channels);
140 143

  
141 144
    if (!(f[0] == f[1])) {
142
        Result res = warning("Explicitly setting parameters to their supposed default values changes the results");
143
        dump(res, f[0], f[1]);
145
        string message = "Explicitly setting parameters to their supposed default values changes the results";
146
        Result res;
147
        if (options & NonDeterministic) res = note(message);
148
        else res = error(message);
149
        if (options & Verbose) dump(res, f[0], f[1]);
144 150
        r.push_back(res);
145 151
    } else {
146 152
        r.push_back(success());
......
150 156
}
151 157

  
152 158
Test::Results
153
TestLengthyConstructor::test(string key)
159
TestLengthyConstructor::test(string key, Options options)
154 160
{
155 161
    time_t t0 = time(0);
156 162
    auto_ptr<Plugin> p(load(key));
TestDefaults.h
47 47
{
48 48
public:
49 49
    TestDefaultProgram() : Test() { }
50
    Results test(std::string key);
50
    Results test(std::string key, Options options);
51 51
    
52 52
protected:
53 53
    static Tester::TestRegistrar<TestDefaultProgram> m_registrar;
......
57 57
{
58 58
public:
59 59
    TestDefaultParameters() : Test() { }
60
    Results test(std::string key);
60
    Results test(std::string key, Options options);
61 61
    
62 62
protected:
63 63
    static Tester::TestRegistrar<TestDefaultParameters> m_registrar;
......
67 67
{
68 68
public:
69 69
    TestLengthyConstructor() : Test() { }
70
    Results test(std::string key);
70
    Results test(std::string key, Options options);
71 71
    
72 72
protected:
73 73
    static Tester::TestRegistrar<TestLengthyConstructor> m_registrar;
TestInputExtremes.cpp
67 67
TestRandomInput::m_registrar("C6 Random input");
68 68

  
69 69
Test::Results
70
TestNormalInput::test(string key)
70
TestNormalInput::test(string key, Options options)
71 71
{
72 72
    Plugin::FeatureSet f;
73 73
    int rate = 44100;
......
95 95
        r.push_back(success());
96 96
    } else {
97 97
        r.push_back(warning("plugin returned one or more NaN/inf values"));
98
        if (options & Verbose) dump(f);
98 99
    }
99 100
    return r;
100 101
}
101 102

  
102 103
Test::Results
103
TestNoInput::test(string key)
104
TestNoInput::test(string key, Options options)
104 105
{
105 106
    auto_ptr<Plugin> p(load(key));
106 107
    Results r;
......
116 117
}
117 118

  
118 119
Test::Results
119
TestShortInput::test(string key)
120
TestShortInput::test(string key, Options options)
120 121
{
121 122
    Plugin::FeatureSet f;
122 123
    int rate = 44100;
......
141 142
        r.push_back(success());
142 143
    } else {
143 144
        r.push_back(warning("plugin returned one or more NaN/inf values"));
145
        if (options & Verbose) dump(f);
144 146
    }
145 147
    return r;
146 148
}
147 149

  
148 150
Test::Results
149
TestSilentInput::test(string key)
151
TestSilentInput::test(string key, Options options)
150 152
{
151 153
    Plugin::FeatureSet f;
152 154
    int rate = 44100;
......
172 174
        r.push_back(success());
173 175
    } else {
174 176
        r.push_back(warning("plugin returned one or more NaN/inf values"));
177
        if (options & Verbose) dump(f);
175 178
    }
176 179
    return r;
177 180
}
178 181

  
179 182
Test::Results
180
TestTooLoudInput::test(string key)
183
TestTooLoudInput::test(string key, Options options)
181 184
{
182 185
    Plugin::FeatureSet f;
183 186
    int rate = 44100;
......
205 208
        r.push_back(success());
206 209
    } else {
207 210
        r.push_back(warning("plugin returned one or more NaN/inf values"));
211
        if (options & Verbose) dump(f);
208 212
    }
209 213
    return r;
210 214
}
211 215

  
212 216
Test::Results
213
TestRandomInput::test(string key)
217
TestRandomInput::test(string key, Options options)
214 218
{
215 219
    Plugin::FeatureSet f;
216 220
    int rate = 44100;
......
238 242
        r.push_back(success());
239 243
    } else {
240 244
        r.push_back(warning("plugin returned one or more NaN/inf values"));
245
        if (options & Verbose) dump(f);
241 246
    }
242 247
    return r;
243 248
}
TestInputExtremes.h
51 51
{
52 52
public:
53 53
    TestNormalInput() : Test() { }
54
    Results test(std::string key);
54
    Results test(std::string key, Options options);
55 55

  
56 56
protected:
57 57
    static Tester::TestRegistrar<TestNormalInput> m_registrar;
......
61 61
{
62 62
public:
63 63
    TestNoInput() : Test() { }
64
    Results test(std::string key);
64
    Results test(std::string key, Options options);
65 65

  
66 66
protected:
67 67
    static Tester::TestRegistrar<TestNoInput> m_registrar;
......
71 71
{
72 72
public:
73 73
    TestShortInput() : Test() { }
74
    Results test(std::string key);
74
    Results test(std::string key, Options options);
75 75

  
76 76
protected:
77 77
    static Tester::TestRegistrar<TestShortInput> m_registrar;
......
81 81
{
82 82
public:
83 83
    TestSilentInput() : Test() { }
84
    Results test(std::string key);
84
    Results test(std::string key, Options options);
85 85

  
86 86
protected:
87 87
    static Tester::TestRegistrar<TestSilentInput> m_registrar;
......
91 91
{
92 92
public:
93 93
    TestTooLoudInput() : Test() { }
94
    Results test(std::string key);
94
    Results test(std::string key, Options options);
95 95

  
96 96
protected:
97 97
    static Tester::TestRegistrar<TestTooLoudInput> m_registrar;
......
101 101
{
102 102
public:
103 103
    TestRandomInput() : Test() { }
104
    Results test(std::string key);
104
    Results test(std::string key, Options options);
105 105

  
106 106
protected:
107 107
    static Tester::TestRegistrar<TestRandomInput> m_registrar;
TestMultipleRuns.cpp
62 62
static const size_t _step = 1000;
63 63

  
64 64
Test::Results
65
TestDistinctRuns::test(string key)
65
TestDistinctRuns::test(string key, Options options)
66 66
{
67 67
    Plugin::FeatureSet f[2];
68 68
    int rate = 44100;
......
89 89
    if (data) destroyTestAudio(data, channels);
90 90

  
91 91
    if (!(f[0] == f[1])) {
92
        Result res = warning("Consecutive runs with separate instances produce different results");
93
        dump(res, f[0], f[1]);
92
        Result res;
93
        string message = "Consecutive runs with separate instances produce different results";
94
        if (options & NonDeterministic) res = note(message);
95
        else res = error(message);
96
        if (options & Verbose) dump(res, f[0], f[1]);
94 97
        r.push_back(res);
95 98
    } else {
96 99
        r.push_back(success());
......
100 103
}
101 104

  
102 105
Test::Results
103
TestReset::test(string key)
106
TestReset::test(string key, Options options)
104 107
{
105 108
    Plugin::FeatureSet f[2];
106 109
    int rate = 44100;
......
129 132
    if (data) destroyTestAudio(data, channels);
130 133

  
131 134
    if (!(f[0] == f[1])) {
132
        Result res = warning("Consecutive runs with the same instance (using reset) produce different results");
133
        dump(res, f[0], f[1]);
135
        string message = "Consecutive runs with the same instance (using reset) produce different results";
136
        Result res;
137
        if (options & NonDeterministic) res = note(message);
138
        else res = error(message);
139
        if (options & Verbose) dump(res, f[0], f[1]);
134 140
        r.push_back(res);
135 141
    } else {
136 142
        r.push_back(success());
......
140 146
}
141 147

  
142 148
Test::Results
143
TestInterleavedRuns::test(string key)
149
TestInterleavedRuns::test(string key, Options options)
144 150
{
145 151
    Plugin::FeatureSet f[2];
146 152
    int rate = 44100;
......
178 184
    if (data) destroyTestAudio(data, channels);
179 185

  
180 186
    if (!(f[0] == f[1])) {
181
        Result res = warning("Simultaneous runs with separate instances produce different results");
182
        dump(res, f[0], f[1]);
187
        string message = "Simultaneous runs with separate instances produce different results";
188
        Result res;
189
        if (options & NonDeterministic) res = note(message);
190
        else res = error(message);
191
        if (options & Verbose) dump(res, f[0], f[1]);
183 192
        r.push_back(res);
184 193
    } else {
185 194
        r.push_back(success());
......
189 198
}
190 199

  
191 200
Test::Results
192
TestDifferentStartTimes::test(string key)
201
TestDifferentStartTimes::test(string key, Options options)
193 202
{
194 203
    Plugin::FeatureSet f[2];
195 204
    int rate = 44100;
......
217 226
    if (data) destroyTestAudio(data, channels);
218 227

  
219 228
    if (f[0] == f[1]) {
220
        Result res = warning("Consecutive runs with different starting timestamps produce the same result");
221
        dump(res, f[0], f[1]);
229
        string message = "Consecutive runs with different starting timestamps produce the same result";
230
        Result res;
231
        if (options & NonDeterministic) res = note(message);
232
        else res = error(message);
233
        if (options & Verbose) dump(res, f[0], f[1]);
222 234
        r.push_back(res);
223 235
    } else {
224 236
        r.push_back(success());
TestMultipleRuns.h
47 47
{
48 48
public:
49 49
    TestDistinctRuns() : Test() { }
50
    Results test(std::string key);
50
    Results test(std::string key, Options options);
51 51
    
52 52
protected:
53 53
    static Tester::TestRegistrar<TestDistinctRuns> m_registrar;
......
57 57
{
58 58
public:
59 59
    TestReset() : Test() { }
60
    Results test(std::string key);
60
    Results test(std::string key, Options options);
61 61
    
62 62
protected:
63 63
    static Tester::TestRegistrar<TestReset> m_registrar;
......
67 67
{
68 68
public:
69 69
    TestInterleavedRuns() : Test() { }
70
    Results test(std::string key);
70
    Results test(std::string key, Options options);
71 71
    
72 72
protected:
73 73
    static Tester::TestRegistrar<TestInterleavedRuns> m_registrar;
......
77 77
{
78 78
public:
79 79
    TestDifferentStartTimes() : Test() { }
80
    Results test(std::string key);
80
    Results test(std::string key, Options options);
81 81
    
82 82
protected:
83 83
    static Tester::TestRegistrar<TestDifferentStartTimes> m_registrar;
TestOutputs.cpp
59 59
static const size_t _step = 1000;
60 60

  
61 61
Test::Results
62
TestOutputNumbers::test(string key)
62
TestOutputNumbers::test(string key, Options options)
63 63
{
64 64
    int rate = 44100;
65 65
    auto_ptr<Plugin> p(load(key, rate));
......
99 99
        }
100 100
    }
101 101
                
102
    if (!r.empty() && (options & Verbose)) dump(f);
102 103
    return r;
103 104
}
104 105

  
105 106
Test::Results
106
TestTimestamps::test(string key)
107
TestTimestamps::test(string key, Options options)
107 108
{
108 109
    int rate = 44100;
109 110

  
......
159 160
        }
160 161
    }
161 162

  
163
    if (!r.empty() && (options & Verbose)) dump(f);
162 164
    return r;
163 165
}
TestOutputs.h
47 47
{
48 48
public:
49 49
    TestOutputNumbers() : Test() { }
50
    Results test(std::string key);
50
    Results test(std::string key, Options options);
51 51
    
52 52
protected:
53 53
    static Tester::TestRegistrar<TestOutputNumbers> m_registrar;
......
57 57
{
58 58
public:
59 59
    TestTimestamps() : Test() { }
60
    Results test(std::string key);
60
    Results test(std::string key, Options options);
61 61
    
62 62
protected:
63 63
    static Tester::TestRegistrar<TestTimestamps> m_registrar;
TestStaticData.cpp
57 57
TestValueRanges::m_registrar("A3 Inappropriate value extents");
58 58

  
59 59
Test::Results
60
TestIdentifiers::test(string key)
60
TestIdentifiers::test(string key, Options options)
61 61
{
62 62
    auto_ptr<Plugin> p(load(key));
63 63
    
......
94 94
}
95 95

  
96 96
Test::Results
97
TestEmptyFields::test(string key)
97
TestEmptyFields::test(string key, Options options)
98 98
{
99 99
    auto_ptr<Plugin> p(load(key));
100 100

  
......
147 147
}
148 148

  
149 149
Test::Results
150
TestValueRanges::test(string key)
150
TestValueRanges::test(string key, Options options)
151 151
{
152 152
    auto_ptr<Plugin> p(load(key));
153 153

  
TestStaticData.h
47 47
{
48 48
public:
49 49
    TestIdentifiers() : Test() { }
50
    Results test(std::string key);
50
    Results test(std::string key, Options options);
51 51
    
52 52
protected:
53 53
    Result testIdentifier(std::string ident, std::string desc);
......
58 58
{
59 59
public:
60 60
    TestEmptyFields() : Test() { }
61
    Results test(std::string key);
61
    Results test(std::string key, Options options);
62 62
    
63 63
protected:
64 64
    Result testMandatory(std::string text, std::string desc);
......
70 70
{
71 71
public:
72 72
    TestValueRanges() : Test() { }
73
    Results test(std::string key);
73
    Results test(std::string key, Options options);
74 74
    
75 75
protected:
76 76
    static Tester::TestRegistrar<TestValueRanges> m_registrar;
Tester.cpp
59 59

  
60 60
using namespace std;
61 61

  
62
Tester::Tester(std::string key) :
63
    m_key(key)
62
Tester::Tester(std::string key, Test::Options options) :
63
    m_key(key),
64
    m_options(options)
64 65
{
65 66
}
66 67

  
......
151 152
            std::cout << " -- Performing test: " << i->first << std::endl;
152 153

  
153 154
            Test *test = i->second->makeTest();
154
            Test::Results results = test->test(m_key);
155
            Test::Results results = test->test(m_key, m_options);
155 156
            delete test;
156 157

  
157 158
            set<string> printed;
Tester.h
47 47
class Tester
48 48
{
49 49
public:
50
    Tester(std::string pluginKey);
50
    Tester(std::string pluginKey, Test::Options);
51 51
    ~Tester();
52 52

  
53 53
    bool test(int &notes, int &warnings, int &errors);
......
71 71
    
72 72
protected:
73 73
    std::string m_key;
74
    Test::Options m_options;
74 75
    typedef std::map<std::string, Registrar *> Registry;
75 76
    static Registry &registry();
76 77
};
vamp-plugin-tester.cpp
44 44
#include <iostream>
45 45

  
46 46
#include <cstdlib>
47
#include <cstring>
47 48

  
48 49
#include "Tester.h"
49 50

  
......
57 58
        "Copyright 2009 QMUL.\n"
58 59
        "Freely redistributable; published under a BSD-style license.\n\n"
59 60
        "Usage:\n"
60
        "  " << name << " [<pluginbasename>:<plugin>]\n\n"
61
        "  " << name << " [-n] [-v] [<pluginbasename>:<plugin>]\n\n"
61 62
        "Example:\n"
62 63
        "  " << name << " vamp-example-plugins:amplitudefollower\n\n"
63 64
        "With an argument, tests one plugin; without, tests all plugins in Vamp path.\n"
64
        "If you have access to a runtime memory checker, you may find it especially\n"
65
        "\nOptions:\n"
66
        "  --nondeterministic, -n    Plugins may be nondeterministic: print a note\n"
67
        "                            instead of an error if results differ between runs\n"
68
        "  --verbose, -v             Show returned features each time a note, warning,\n"
69
        "                            or error arises from feature data\n"
70
        "\nIf you have access to a runtime memory checker, you may find it especially\n"
65 71
        "helpful to run this tester under it and watch for errors thus provoked.\n"
66 72
         << endl;
67 73
    exit(2);
......
76 82
        else ++scooter;
77 83
    }
78 84
    if (!name || !*name) name = argv[0];
85

  
86
    bool nondeterministic = false;
87
    bool verbose = false;
88
    string argument;
89
    for (int i = 1; i < argc; ++i) {
90
        if (!argv[i]) break;
91
        if (argv[i][0] == '-') {
92
            if (!strcmp(argv[i], "-v") ||
93
                !strcmp(argv[i], "--verbose")) {
94
                verbose = 1;
95
                continue;
96
            }
97
            if (!strcmp(argv[i], "-n") ||
98
                !strcmp(argv[i], "--nondeterministic")) {
99
                nondeterministic = 1;
100
                continue;
101
            }
102
            usage(name);
103
        } else {
104
            if (argument != "") usage(name);
105
            else argument = argv[i];
106
        }
107
    }
79 108
    
80
    if (argc > 2) usage(name);
81
    if (argc == 2 && argv[1][0] == '-') usage(name);
82

  
83 109
    cerr << name << ": Running..." << endl;
84 110

  
85
    if (argc == 1) {
111
    Test::Options opts = Test::NoOption;
112
    if (nondeterministic) opts |= Test::NonDeterministic;
113
    if (verbose) opts |= Test::Verbose;
114

  
115
    if (argument == "") {
86 116
        bool good = true;
87 117
        Vamp::HostExt::PluginLoader::PluginKeyList keys =
88 118
            Vamp::HostExt::PluginLoader::getInstance()->listPlugins();
89 119
        int notes = 0, warnings = 0, errors = 0;
90 120
        for (int i = 0; i < (int)keys.size(); ++i) {
91 121
            cout << "Testing plugin: " << keys[i] << endl;
92
            Tester tester(keys[i]);
122
            Tester tester(keys[i], opts);
93 123
            if (tester.test(notes, warnings, errors)) {
94 124
                cout << name << ": All tests succeeded for this plugin" << endl;
95 125
            } else {
......
115 145
            return 1;
116 146
        }   
117 147
    } else {
118
        string key = argv[1];
119
        Tester tester(key);
148
        string key = argument;
149
        Tester tester(key, opts);
120 150
        int notes = 0, warnings = 0, errors = 0;
121 151
        if (tester.test(notes, warnings, errors)) {
122 152
            cout << name << ": All tests succeeded";

Also available in: Unified diff