Chris@0
|
1
|
Chris@49
|
2 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@0
|
3
|
Chris@0
|
4 /*
|
Chris@52
|
5 Sonic Visualiser
|
Chris@52
|
6 An audio file viewer and annotation editor.
|
Chris@52
|
7 Centre for Digital Music, Queen Mary, University of London.
|
Chris@52
|
8 This file copyright 2006 Chris Cannam.
|
Chris@0
|
9
|
Chris@52
|
10 This program is free software; you can redistribute it and/or
|
Chris@52
|
11 modify it under the terms of the GNU General Public License as
|
Chris@52
|
12 published by the Free Software Foundation; either version 2 of the
|
Chris@52
|
13 License, or (at your option) any later version. See the file
|
Chris@52
|
14 COPYING included with this distribution for more information.
|
Chris@0
|
15 */
|
Chris@0
|
16
|
Chris@0
|
17 #include "FeatureExtractionPluginTransform.h"
|
Chris@0
|
18
|
Chris@0
|
19 #include "plugin/FeatureExtractionPluginFactory.h"
|
Chris@66
|
20 #include "plugin/PluginXml.h"
|
Chris@66
|
21 #include "vamp-sdk/Plugin.h"
|
Chris@0
|
22
|
Chris@0
|
23 #include "base/Model.h"
|
Chris@0
|
24 #include "model/SparseOneDimensionalModel.h"
|
Chris@0
|
25 #include "model/SparseTimeValueModel.h"
|
Chris@0
|
26 #include "model/DenseThreeDimensionalModel.h"
|
Chris@0
|
27 #include "model/DenseTimeValueModel.h"
|
Chris@0
|
28
|
Chris@0
|
29 #include <iostream>
|
Chris@0
|
30
|
Chris@0
|
31 FeatureExtractionPluginTransform::FeatureExtractionPluginTransform(Model *inputModel,
|
Chris@0
|
32 QString pluginId,
|
Chris@64
|
33 int channel,
|
Chris@56
|
34 QString configurationXml,
|
Chris@0
|
35 QString outputName) :
|
Chris@0
|
36 Transform(inputModel),
|
Chris@0
|
37 m_plugin(0),
|
Chris@64
|
38 m_channel(channel),
|
Chris@0
|
39 m_descriptor(0),
|
Chris@0
|
40 m_outputFeatureNo(0)
|
Chris@0
|
41 {
|
Chris@0
|
42 std::cerr << "FeatureExtractionPluginTransform::FeatureExtractionPluginTransform: plugin " << pluginId.toStdString() << ", outputName " << outputName.toStdString() << std::endl;
|
Chris@0
|
43
|
Chris@0
|
44 FeatureExtractionPluginFactory *factory =
|
Chris@0
|
45 FeatureExtractionPluginFactory::instanceFor(pluginId);
|
Chris@0
|
46
|
Chris@0
|
47 if (!factory) {
|
Chris@0
|
48 std::cerr << "FeatureExtractionPluginTransform: No factory available for plugin id \""
|
Chris@0
|
49 << pluginId.toStdString() << "\"" << std::endl;
|
Chris@0
|
50 return;
|
Chris@0
|
51 }
|
Chris@0
|
52
|
Chris@0
|
53 m_plugin = factory->instantiatePlugin(pluginId, m_input->getSampleRate());
|
Chris@0
|
54
|
Chris@0
|
55 if (!m_plugin) {
|
Chris@0
|
56 std::cerr << "FeatureExtractionPluginTransform: Failed to instantiate plugin \""
|
Chris@0
|
57 << pluginId.toStdString() << "\"" << std::endl;
|
Chris@0
|
58 return;
|
Chris@0
|
59 }
|
Chris@0
|
60
|
Chris@56
|
61 if (configurationXml != "") {
|
Chris@66
|
62 PluginXml(m_plugin).setParametersFromXml(configurationXml);
|
Chris@56
|
63 }
|
Chris@56
|
64
|
Chris@66
|
65 Vamp::Plugin::OutputList outputs =
|
Chris@0
|
66 m_plugin->getOutputDescriptors();
|
Chris@0
|
67
|
Chris@0
|
68 if (outputs.empty()) {
|
Chris@0
|
69 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
|
Chris@0
|
70 << pluginId.toStdString() << "\" has no outputs" << std::endl;
|
Chris@0
|
71 return;
|
Chris@0
|
72 }
|
Chris@0
|
73
|
Chris@0
|
74 for (size_t i = 0; i < outputs.size(); ++i) {
|
Chris@0
|
75 if (outputName == "" || outputs[i].name == outputName.toStdString()) {
|
Chris@0
|
76 m_outputFeatureNo = i;
|
Chris@66
|
77 m_descriptor = new Vamp::Plugin::OutputDescriptor
|
Chris@0
|
78 (outputs[i]);
|
Chris@0
|
79 break;
|
Chris@0
|
80 }
|
Chris@0
|
81 }
|
Chris@0
|
82
|
Chris@0
|
83 if (!m_descriptor) {
|
Chris@0
|
84 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
|
Chris@0
|
85 << pluginId.toStdString() << "\" has no output named \""
|
Chris@0
|
86 << outputName.toStdString() << "\"" << std::endl;
|
Chris@0
|
87 return;
|
Chris@0
|
88 }
|
Chris@0
|
89
|
Chris@0
|
90 std::cerr << "FeatureExtractionPluginTransform: output sample type "
|
Chris@0
|
91 << m_descriptor->sampleType << std::endl;
|
Chris@0
|
92
|
Chris@0
|
93 int valueCount = 1;
|
Chris@0
|
94 float minValue = 0.0, maxValue = 0.0;
|
Chris@0
|
95
|
Chris@0
|
96 if (m_descriptor->hasFixedValueCount) {
|
Chris@0
|
97 valueCount = m_descriptor->valueCount;
|
Chris@0
|
98 }
|
Chris@0
|
99
|
Chris@0
|
100 if (valueCount > 0 && m_descriptor->hasKnownExtents) {
|
Chris@0
|
101 minValue = m_descriptor->minValue;
|
Chris@0
|
102 maxValue = m_descriptor->maxValue;
|
Chris@0
|
103 }
|
Chris@0
|
104
|
Chris@0
|
105 size_t modelRate = m_input->getSampleRate();
|
Chris@0
|
106 size_t modelResolution = 1;
|
Chris@0
|
107
|
Chris@0
|
108 switch (m_descriptor->sampleType) {
|
Chris@0
|
109
|
Chris@66
|
110 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
|
Chris@0
|
111 if (m_descriptor->sampleRate != 0.0) {
|
Chris@0
|
112 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
|
Chris@0
|
113 }
|
Chris@0
|
114 break;
|
Chris@0
|
115
|
Chris@66
|
116 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
|
Chris@0
|
117 modelResolution = m_plugin->getPreferredStepSize();
|
Chris@0
|
118 break;
|
Chris@0
|
119
|
Chris@66
|
120 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
|
Chris@0
|
121 modelRate = m_descriptor->sampleRate;
|
Chris@0
|
122 break;
|
Chris@0
|
123 }
|
Chris@0
|
124
|
Chris@0
|
125 if (valueCount == 0) {
|
Chris@0
|
126
|
Chris@20
|
127 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
|
Chris@20
|
128 false);
|
Chris@0
|
129
|
Chris@0
|
130 } else if (valueCount == 1 ||
|
Chris@0
|
131
|
Chris@0
|
132 // We don't have a sparse 3D model
|
Chris@0
|
133 m_descriptor->sampleType ==
|
Chris@66
|
134 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
135
|
Chris@63
|
136 SparseTimeValueModel *model = new SparseTimeValueModel
|
Chris@63
|
137 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@63
|
138 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
|
Chris@63
|
139
|
Chris@63
|
140 m_output = model;
|
Chris@0
|
141
|
Chris@0
|
142 } else {
|
Chris@0
|
143
|
Chris@0
|
144 m_output = new DenseThreeDimensionalModel(modelRate, modelResolution,
|
Chris@19
|
145 valueCount, false);
|
Chris@20
|
146
|
Chris@20
|
147 if (!m_descriptor->valueNames.empty()) {
|
Chris@20
|
148 std::vector<QString> names;
|
Chris@20
|
149 for (size_t i = 0; i < m_descriptor->valueNames.size(); ++i) {
|
Chris@20
|
150 names.push_back(m_descriptor->valueNames[i].c_str());
|
Chris@20
|
151 }
|
Chris@20
|
152 (dynamic_cast<DenseThreeDimensionalModel *>(m_output))
|
Chris@20
|
153 ->setBinNames(names);
|
Chris@20
|
154 }
|
Chris@0
|
155 }
|
Chris@0
|
156 }
|
Chris@0
|
157
|
Chris@0
|
158 FeatureExtractionPluginTransform::~FeatureExtractionPluginTransform()
|
Chris@0
|
159 {
|
Chris@0
|
160 delete m_plugin;
|
Chris@0
|
161 delete m_descriptor;
|
Chris@0
|
162 }
|
Chris@0
|
163
|
Chris@0
|
164 DenseTimeValueModel *
|
Chris@0
|
165 FeatureExtractionPluginTransform::getInput()
|
Chris@0
|
166 {
|
Chris@0
|
167 DenseTimeValueModel *dtvm =
|
Chris@0
|
168 dynamic_cast<DenseTimeValueModel *>(getInputModel());
|
Chris@0
|
169 if (!dtvm) {
|
Chris@0
|
170 std::cerr << "FeatureExtractionPluginTransform::getInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
|
Chris@0
|
171 }
|
Chris@0
|
172 return dtvm;
|
Chris@0
|
173 }
|
Chris@0
|
174
|
Chris@0
|
175 void
|
Chris@0
|
176 FeatureExtractionPluginTransform::run()
|
Chris@0
|
177 {
|
Chris@0
|
178 DenseTimeValueModel *input = getInput();
|
Chris@0
|
179 if (!input) return;
|
Chris@0
|
180
|
Chris@0
|
181 if (!m_output) return;
|
Chris@0
|
182
|
Chris@0
|
183 size_t channelCount = input->getChannelCount();
|
Chris@0
|
184 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@0
|
185 channelCount = 1;
|
Chris@0
|
186 }
|
Chris@0
|
187 if (m_plugin->getMinChannelCount() > channelCount) {
|
Chris@0
|
188 std::cerr << "FeatureExtractionPluginTransform::run: "
|
Chris@0
|
189 << "Can't provide enough channels to plugin (plugin min "
|
Chris@0
|
190 << m_plugin->getMinChannelCount() << ", max "
|
Chris@0
|
191 << m_plugin->getMaxChannelCount() << ", input model has "
|
Chris@0
|
192 << input->getChannelCount() << ")" << std::endl;
|
Chris@0
|
193 return;
|
Chris@0
|
194 }
|
Chris@0
|
195
|
Chris@0
|
196 size_t sampleRate = m_input->getSampleRate();
|
Chris@0
|
197
|
Chris@0
|
198 size_t stepSize = m_plugin->getPreferredStepSize();
|
Chris@0
|
199 size_t blockSize = m_plugin->getPreferredBlockSize();
|
Chris@0
|
200
|
Chris@0
|
201 m_plugin->initialise(channelCount, stepSize, blockSize);
|
Chris@0
|
202
|
Chris@0
|
203 float **buffers = new float*[channelCount];
|
Chris@0
|
204 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@0
|
205 buffers[ch] = new float[blockSize];
|
Chris@0
|
206 }
|
Chris@0
|
207
|
Chris@0
|
208 size_t startFrame = m_input->getStartFrame();
|
Chris@0
|
209 size_t endFrame = m_input->getEndFrame();
|
Chris@0
|
210 size_t blockFrame = startFrame;
|
Chris@0
|
211
|
Chris@0
|
212 size_t prevCompletion = 0;
|
Chris@0
|
213
|
Chris@0
|
214 while (blockFrame < endFrame) {
|
Chris@0
|
215
|
Chris@0
|
216 // std::cerr << "FeatureExtractionPluginTransform::run: blockFrame "
|
Chris@0
|
217 // << blockFrame << std::endl;
|
Chris@0
|
218
|
Chris@0
|
219 size_t completion =
|
Chris@0
|
220 (((blockFrame - startFrame) / stepSize) * 99) /
|
Chris@0
|
221 ( (endFrame - startFrame) / stepSize);
|
Chris@0
|
222
|
Chris@0
|
223 // channelCount is either m_input->channelCount or 1
|
Chris@0
|
224
|
Chris@0
|
225 size_t got = 0;
|
Chris@0
|
226
|
Chris@0
|
227 if (channelCount == 1) {
|
Chris@0
|
228 got = input->getValues
|
Chris@64
|
229 (m_channel, blockFrame, blockFrame + blockSize, buffers[0]);
|
Chris@0
|
230 while (got < blockSize) {
|
Chris@0
|
231 buffers[0][got++] = 0.0;
|
Chris@0
|
232 }
|
Chris@0
|
233 } else {
|
Chris@0
|
234 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@0
|
235 got = input->getValues
|
Chris@0
|
236 (ch, blockFrame, blockFrame + blockSize, buffers[ch]);
|
Chris@0
|
237 while (got < blockSize) {
|
Chris@0
|
238 buffers[ch][got++] = 0.0;
|
Chris@0
|
239 }
|
Chris@0
|
240 }
|
Chris@0
|
241 }
|
Chris@0
|
242
|
Chris@66
|
243 Vamp::Plugin::FeatureSet features = m_plugin->process
|
Chris@66
|
244 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
|
Chris@0
|
245
|
Chris@0
|
246 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@66
|
247 Vamp::Plugin::Feature feature =
|
Chris@0
|
248 features[m_outputFeatureNo][fi];
|
Chris@0
|
249 addFeature(blockFrame, feature);
|
Chris@0
|
250 }
|
Chris@0
|
251
|
Chris@0
|
252 if (blockFrame == startFrame || completion > prevCompletion) {
|
Chris@0
|
253 setCompletion(completion);
|
Chris@0
|
254 prevCompletion = completion;
|
Chris@0
|
255 }
|
Chris@0
|
256
|
Chris@0
|
257 blockFrame += stepSize;
|
Chris@0
|
258 }
|
Chris@0
|
259
|
Chris@66
|
260 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
|
Chris@0
|
261
|
Chris@0
|
262 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@66
|
263 Vamp::Plugin::Feature feature =
|
Chris@0
|
264 features[m_outputFeatureNo][fi];
|
Chris@0
|
265 addFeature(blockFrame, feature);
|
Chris@0
|
266 }
|
Chris@0
|
267
|
Chris@0
|
268 setCompletion(100);
|
Chris@0
|
269 }
|
Chris@0
|
270
|
Chris@0
|
271
|
Chris@0
|
272 void
|
Chris@0
|
273 FeatureExtractionPluginTransform::addFeature(size_t blockFrame,
|
Chris@66
|
274 const Vamp::Plugin::Feature &feature)
|
Chris@0
|
275 {
|
Chris@0
|
276 size_t inputRate = m_input->getSampleRate();
|
Chris@0
|
277
|
Chris@0
|
278 // std::cerr << "FeatureExtractionPluginTransform::addFeature("
|
Chris@0
|
279 // << blockFrame << ")" << std::endl;
|
Chris@0
|
280
|
Chris@0
|
281 int valueCount = 1;
|
Chris@0
|
282 if (m_descriptor->hasFixedValueCount) {
|
Chris@0
|
283 valueCount = m_descriptor->valueCount;
|
Chris@0
|
284 }
|
Chris@0
|
285
|
Chris@0
|
286 size_t frame = blockFrame;
|
Chris@0
|
287
|
Chris@0
|
288 if (m_descriptor->sampleType ==
|
Chris@66
|
289 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
290
|
Chris@0
|
291 if (!feature.hasTimestamp) {
|
Chris@0
|
292 std::cerr
|
Chris@0
|
293 << "WARNING: FeatureExtractionPluginTransform::addFeature: "
|
Chris@0
|
294 << "Feature has variable sample rate but no timestamp!"
|
Chris@0
|
295 << std::endl;
|
Chris@0
|
296 return;
|
Chris@0
|
297 } else {
|
Chris@66
|
298 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
|
Chris@0
|
299 }
|
Chris@0
|
300
|
Chris@0
|
301 } else if (m_descriptor->sampleType ==
|
Chris@66
|
302 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
|
Chris@0
|
303
|
Chris@0
|
304 if (feature.hasTimestamp) {
|
Chris@0
|
305 //!!! warning: sampleRate may be non-integral
|
Chris@66
|
306 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
|
Chris@66
|
307 m_descriptor->sampleRate);
|
Chris@0
|
308 } else {
|
Chris@0
|
309 frame = m_output->getEndFrame() + 1;
|
Chris@0
|
310 }
|
Chris@0
|
311 }
|
Chris@0
|
312
|
Chris@0
|
313 if (valueCount == 0) {
|
Chris@0
|
314
|
Chris@0
|
315 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
|
Chris@0
|
316 if (!model) return;
|
Chris@0
|
317 model->addPoint(SparseOneDimensionalModel::Point(frame, feature.label.c_str()));
|
Chris@0
|
318
|
Chris@0
|
319 } else if (valueCount == 1 ||
|
Chris@0
|
320 m_descriptor->sampleType ==
|
Chris@66
|
321 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
322
|
Chris@0
|
323 float value = 0.0;
|
Chris@0
|
324 if (feature.values.size() > 0) value = feature.values[0];
|
Chris@0
|
325
|
Chris@0
|
326 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
|
Chris@0
|
327 if (!model) return;
|
Chris@0
|
328 model->addPoint(SparseTimeValueModel::Point(frame, value, feature.label.c_str()));
|
Chris@0
|
329
|
Chris@0
|
330 } else {
|
Chris@0
|
331
|
Chris@0
|
332 DenseThreeDimensionalModel::BinValueSet values = feature.values;
|
Chris@0
|
333
|
Chris@0
|
334 DenseThreeDimensionalModel *model = getOutput<DenseThreeDimensionalModel>();
|
Chris@0
|
335 if (!model) return;
|
Chris@0
|
336
|
Chris@0
|
337 model->setBinValues(frame, values);
|
Chris@0
|
338 }
|
Chris@0
|
339 }
|
Chris@0
|
340
|
Chris@0
|
341 void
|
Chris@0
|
342 FeatureExtractionPluginTransform::setCompletion(int completion)
|
Chris@0
|
343 {
|
Chris@0
|
344 int valueCount = 1;
|
Chris@0
|
345 if (m_descriptor->hasFixedValueCount) {
|
Chris@0
|
346 valueCount = m_descriptor->valueCount;
|
Chris@0
|
347 }
|
Chris@0
|
348
|
Chris@0
|
349 if (valueCount == 0) {
|
Chris@0
|
350
|
Chris@0
|
351 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
|
Chris@0
|
352 if (!model) return;
|
Chris@0
|
353 model->setCompletion(completion);
|
Chris@0
|
354
|
Chris@0
|
355 } else if (valueCount == 1 ||
|
Chris@0
|
356 m_descriptor->sampleType ==
|
Chris@66
|
357 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
358
|
Chris@0
|
359 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
|
Chris@0
|
360 if (!model) return;
|
Chris@0
|
361 model->setCompletion(completion);
|
Chris@0
|
362
|
Chris@0
|
363 } else {
|
Chris@0
|
364
|
Chris@19
|
365 DenseThreeDimensionalModel *model = getOutput<DenseThreeDimensionalModel>();
|
Chris@19
|
366 if (!model) return;
|
Chris@19
|
367 model->setCompletion(completion);
|
Chris@0
|
368 }
|
Chris@0
|
369 }
|
Chris@0
|
370
|