Chris@0
|
1
|
Chris@49
|
2 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@0
|
3
|
Chris@0
|
4 /*
|
Chris@52
|
5 Sonic Visualiser
|
Chris@52
|
6 An audio file viewer and annotation editor.
|
Chris@52
|
7 Centre for Digital Music, Queen Mary, University of London.
|
Chris@52
|
8 This file copyright 2006 Chris Cannam.
|
Chris@0
|
9
|
Chris@52
|
10 This program is free software; you can redistribute it and/or
|
Chris@52
|
11 modify it under the terms of the GNU General Public License as
|
Chris@52
|
12 published by the Free Software Foundation; either version 2 of the
|
Chris@52
|
13 License, or (at your option) any later version. See the file
|
Chris@52
|
14 COPYING included with this distribution for more information.
|
Chris@0
|
15 */
|
Chris@0
|
16
|
Chris@0
|
17 #include "FeatureExtractionPluginTransform.h"
|
Chris@0
|
18
|
Chris@0
|
19 #include "plugin/FeatureExtractionPluginFactory.h"
|
Chris@66
|
20 #include "plugin/PluginXml.h"
|
Chris@66
|
21 #include "vamp-sdk/Plugin.h"
|
Chris@0
|
22
|
Chris@0
|
23 #include "base/Model.h"
|
Chris@67
|
24 #include "base/Window.h"
|
Chris@0
|
25 #include "model/SparseOneDimensionalModel.h"
|
Chris@0
|
26 #include "model/SparseTimeValueModel.h"
|
Chris@0
|
27 #include "model/DenseThreeDimensionalModel.h"
|
Chris@0
|
28 #include "model/DenseTimeValueModel.h"
|
Chris@0
|
29
|
Chris@67
|
30 #include <fftw3.h>
|
Chris@67
|
31
|
Chris@0
|
32 #include <iostream>
|
Chris@0
|
33
|
Chris@0
|
34 FeatureExtractionPluginTransform::FeatureExtractionPluginTransform(Model *inputModel,
|
Chris@0
|
35 QString pluginId,
|
Chris@64
|
36 int channel,
|
Chris@56
|
37 QString configurationXml,
|
Chris@0
|
38 QString outputName) :
|
Chris@0
|
39 Transform(inputModel),
|
Chris@0
|
40 m_plugin(0),
|
Chris@64
|
41 m_channel(channel),
|
Chris@0
|
42 m_descriptor(0),
|
Chris@0
|
43 m_outputFeatureNo(0)
|
Chris@0
|
44 {
|
Chris@0
|
45 std::cerr << "FeatureExtractionPluginTransform::FeatureExtractionPluginTransform: plugin " << pluginId.toStdString() << ", outputName " << outputName.toStdString() << std::endl;
|
Chris@0
|
46
|
Chris@0
|
47 FeatureExtractionPluginFactory *factory =
|
Chris@0
|
48 FeatureExtractionPluginFactory::instanceFor(pluginId);
|
Chris@0
|
49
|
Chris@0
|
50 if (!factory) {
|
Chris@0
|
51 std::cerr << "FeatureExtractionPluginTransform: No factory available for plugin id \""
|
Chris@0
|
52 << pluginId.toStdString() << "\"" << std::endl;
|
Chris@0
|
53 return;
|
Chris@0
|
54 }
|
Chris@0
|
55
|
Chris@0
|
56 m_plugin = factory->instantiatePlugin(pluginId, m_input->getSampleRate());
|
Chris@0
|
57
|
Chris@0
|
58 if (!m_plugin) {
|
Chris@0
|
59 std::cerr << "FeatureExtractionPluginTransform: Failed to instantiate plugin \""
|
Chris@0
|
60 << pluginId.toStdString() << "\"" << std::endl;
|
Chris@0
|
61 return;
|
Chris@0
|
62 }
|
Chris@0
|
63
|
Chris@56
|
64 if (configurationXml != "") {
|
Chris@66
|
65 PluginXml(m_plugin).setParametersFromXml(configurationXml);
|
Chris@56
|
66 }
|
Chris@56
|
67
|
Chris@66
|
68 Vamp::Plugin::OutputList outputs =
|
Chris@0
|
69 m_plugin->getOutputDescriptors();
|
Chris@0
|
70
|
Chris@0
|
71 if (outputs.empty()) {
|
Chris@0
|
72 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
|
Chris@0
|
73 << pluginId.toStdString() << "\" has no outputs" << std::endl;
|
Chris@0
|
74 return;
|
Chris@0
|
75 }
|
Chris@0
|
76
|
Chris@0
|
77 for (size_t i = 0; i < outputs.size(); ++i) {
|
Chris@0
|
78 if (outputName == "" || outputs[i].name == outputName.toStdString()) {
|
Chris@0
|
79 m_outputFeatureNo = i;
|
Chris@66
|
80 m_descriptor = new Vamp::Plugin::OutputDescriptor
|
Chris@0
|
81 (outputs[i]);
|
Chris@0
|
82 break;
|
Chris@0
|
83 }
|
Chris@0
|
84 }
|
Chris@0
|
85
|
Chris@0
|
86 if (!m_descriptor) {
|
Chris@0
|
87 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
|
Chris@0
|
88 << pluginId.toStdString() << "\" has no output named \""
|
Chris@0
|
89 << outputName.toStdString() << "\"" << std::endl;
|
Chris@0
|
90 return;
|
Chris@0
|
91 }
|
Chris@0
|
92
|
Chris@0
|
93 std::cerr << "FeatureExtractionPluginTransform: output sample type "
|
Chris@0
|
94 << m_descriptor->sampleType << std::endl;
|
Chris@0
|
95
|
Chris@0
|
96 int valueCount = 1;
|
Chris@0
|
97 float minValue = 0.0, maxValue = 0.0;
|
Chris@0
|
98
|
Chris@0
|
99 if (m_descriptor->hasFixedValueCount) {
|
Chris@0
|
100 valueCount = m_descriptor->valueCount;
|
Chris@0
|
101 }
|
Chris@0
|
102
|
Chris@0
|
103 if (valueCount > 0 && m_descriptor->hasKnownExtents) {
|
Chris@0
|
104 minValue = m_descriptor->minValue;
|
Chris@0
|
105 maxValue = m_descriptor->maxValue;
|
Chris@0
|
106 }
|
Chris@0
|
107
|
Chris@0
|
108 size_t modelRate = m_input->getSampleRate();
|
Chris@0
|
109 size_t modelResolution = 1;
|
Chris@0
|
110
|
Chris@0
|
111 switch (m_descriptor->sampleType) {
|
Chris@0
|
112
|
Chris@66
|
113 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
|
Chris@0
|
114 if (m_descriptor->sampleRate != 0.0) {
|
Chris@0
|
115 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
|
Chris@0
|
116 }
|
Chris@0
|
117 break;
|
Chris@0
|
118
|
Chris@66
|
119 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
|
Chris@0
|
120 modelResolution = m_plugin->getPreferredStepSize();
|
Chris@0
|
121 break;
|
Chris@0
|
122
|
Chris@66
|
123 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
|
Chris@0
|
124 modelRate = m_descriptor->sampleRate;
|
Chris@0
|
125 break;
|
Chris@0
|
126 }
|
Chris@0
|
127
|
Chris@0
|
128 if (valueCount == 0) {
|
Chris@0
|
129
|
Chris@20
|
130 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
|
Chris@20
|
131 false);
|
Chris@0
|
132
|
Chris@0
|
133 } else if (valueCount == 1 ||
|
Chris@0
|
134
|
Chris@0
|
135 // We don't have a sparse 3D model
|
Chris@0
|
136 m_descriptor->sampleType ==
|
Chris@66
|
137 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
138
|
Chris@63
|
139 SparseTimeValueModel *model = new SparseTimeValueModel
|
Chris@63
|
140 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@63
|
141 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
|
Chris@63
|
142
|
Chris@63
|
143 m_output = model;
|
Chris@0
|
144
|
Chris@0
|
145 } else {
|
Chris@0
|
146
|
Chris@0
|
147 m_output = new DenseThreeDimensionalModel(modelRate, modelResolution,
|
Chris@19
|
148 valueCount, false);
|
Chris@20
|
149
|
Chris@20
|
150 if (!m_descriptor->valueNames.empty()) {
|
Chris@20
|
151 std::vector<QString> names;
|
Chris@20
|
152 for (size_t i = 0; i < m_descriptor->valueNames.size(); ++i) {
|
Chris@20
|
153 names.push_back(m_descriptor->valueNames[i].c_str());
|
Chris@20
|
154 }
|
Chris@20
|
155 (dynamic_cast<DenseThreeDimensionalModel *>(m_output))
|
Chris@20
|
156 ->setBinNames(names);
|
Chris@20
|
157 }
|
Chris@0
|
158 }
|
Chris@0
|
159 }
|
Chris@0
|
160
|
Chris@0
|
161 FeatureExtractionPluginTransform::~FeatureExtractionPluginTransform()
|
Chris@0
|
162 {
|
Chris@0
|
163 delete m_plugin;
|
Chris@0
|
164 delete m_descriptor;
|
Chris@0
|
165 }
|
Chris@0
|
166
|
Chris@0
|
167 DenseTimeValueModel *
|
Chris@0
|
168 FeatureExtractionPluginTransform::getInput()
|
Chris@0
|
169 {
|
Chris@0
|
170 DenseTimeValueModel *dtvm =
|
Chris@0
|
171 dynamic_cast<DenseTimeValueModel *>(getInputModel());
|
Chris@0
|
172 if (!dtvm) {
|
Chris@0
|
173 std::cerr << "FeatureExtractionPluginTransform::getInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
|
Chris@0
|
174 }
|
Chris@0
|
175 return dtvm;
|
Chris@0
|
176 }
|
Chris@0
|
177
|
Chris@0
|
178 void
|
Chris@0
|
179 FeatureExtractionPluginTransform::run()
|
Chris@0
|
180 {
|
Chris@0
|
181 DenseTimeValueModel *input = getInput();
|
Chris@0
|
182 if (!input) return;
|
Chris@0
|
183
|
Chris@0
|
184 if (!m_output) return;
|
Chris@0
|
185
|
Chris@0
|
186 size_t channelCount = input->getChannelCount();
|
Chris@0
|
187 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@0
|
188 channelCount = 1;
|
Chris@0
|
189 }
|
Chris@0
|
190 if (m_plugin->getMinChannelCount() > channelCount) {
|
Chris@0
|
191 std::cerr << "FeatureExtractionPluginTransform::run: "
|
Chris@0
|
192 << "Can't provide enough channels to plugin (plugin min "
|
Chris@0
|
193 << m_plugin->getMinChannelCount() << ", max "
|
Chris@0
|
194 << m_plugin->getMaxChannelCount() << ", input model has "
|
Chris@0
|
195 << input->getChannelCount() << ")" << std::endl;
|
Chris@0
|
196 return;
|
Chris@0
|
197 }
|
Chris@0
|
198
|
Chris@0
|
199 size_t sampleRate = m_input->getSampleRate();
|
Chris@0
|
200
|
Chris@0
|
201 size_t stepSize = m_plugin->getPreferredStepSize();
|
Chris@0
|
202 size_t blockSize = m_plugin->getPreferredBlockSize();
|
Chris@0
|
203
|
Chris@0
|
204 m_plugin->initialise(channelCount, stepSize, blockSize);
|
Chris@0
|
205
|
Chris@0
|
206 float **buffers = new float*[channelCount];
|
Chris@0
|
207 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@0
|
208 buffers[ch] = new float[blockSize];
|
Chris@0
|
209 }
|
Chris@0
|
210
|
Chris@67
|
211 double *fftInput = 0;
|
Chris@67
|
212 fftw_complex *fftOutput = 0;
|
Chris@67
|
213 fftw_plan fftPlan = 0;
|
Chris@67
|
214 Window<double> windower(HanningWindow, blockSize);
|
Chris@67
|
215
|
Chris@67
|
216 if (m_plugin->getInputDomain() == Vamp::Plugin::FrequencyDomain) {
|
Chris@67
|
217
|
Chris@67
|
218 fftInput = (double *)fftw_malloc(blockSize * sizeof(double));
|
Chris@67
|
219 fftOutput = (fftw_complex *)fftw_malloc(blockSize * sizeof(fftw_complex));
|
Chris@67
|
220 fftPlan = fftw_plan_dft_r2c_1d(blockSize, fftInput, fftOutput,
|
Chris@67
|
221 FFTW_ESTIMATE);
|
Chris@67
|
222 if (!fftPlan) {
|
Chris@67
|
223 std::cerr << "ERROR: FeatureExtractionPluginTransform::run(): fftw_plan failed! Results will be garbage" << std::endl;
|
Chris@67
|
224 }
|
Chris@67
|
225 }
|
Chris@67
|
226
|
Chris@0
|
227 size_t startFrame = m_input->getStartFrame();
|
Chris@0
|
228 size_t endFrame = m_input->getEndFrame();
|
Chris@0
|
229 size_t blockFrame = startFrame;
|
Chris@0
|
230
|
Chris@0
|
231 size_t prevCompletion = 0;
|
Chris@0
|
232
|
Chris@0
|
233 while (blockFrame < endFrame) {
|
Chris@0
|
234
|
Chris@0
|
235 // std::cerr << "FeatureExtractionPluginTransform::run: blockFrame "
|
Chris@0
|
236 // << blockFrame << std::endl;
|
Chris@0
|
237
|
Chris@0
|
238 size_t completion =
|
Chris@0
|
239 (((blockFrame - startFrame) / stepSize) * 99) /
|
Chris@0
|
240 ( (endFrame - startFrame) / stepSize);
|
Chris@0
|
241
|
Chris@0
|
242 // channelCount is either m_input->channelCount or 1
|
Chris@0
|
243
|
Chris@0
|
244 size_t got = 0;
|
Chris@0
|
245
|
Chris@0
|
246 if (channelCount == 1) {
|
Chris@0
|
247 got = input->getValues
|
Chris@64
|
248 (m_channel, blockFrame, blockFrame + blockSize, buffers[0]);
|
Chris@0
|
249 while (got < blockSize) {
|
Chris@0
|
250 buffers[0][got++] = 0.0;
|
Chris@0
|
251 }
|
Chris@0
|
252 } else {
|
Chris@0
|
253 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@0
|
254 got = input->getValues
|
Chris@0
|
255 (ch, blockFrame, blockFrame + blockSize, buffers[ch]);
|
Chris@0
|
256 while (got < blockSize) {
|
Chris@0
|
257 buffers[ch][got++] = 0.0;
|
Chris@0
|
258 }
|
Chris@0
|
259 }
|
Chris@0
|
260 }
|
Chris@0
|
261
|
Chris@67
|
262 if (fftPlan) {
|
Chris@67
|
263 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@67
|
264 for (size_t i = 0; i < blockSize; ++i) {
|
Chris@67
|
265 fftInput[i] = buffers[ch][i];
|
Chris@67
|
266 }
|
Chris@67
|
267 windower.cut(fftInput);
|
Chris@67
|
268 for (size_t i = 0; i < blockSize/2; ++i) {
|
Chris@67
|
269 double temp = fftInput[i];
|
Chris@67
|
270 fftInput[i] = fftInput[i + blockSize/2];
|
Chris@67
|
271 fftInput[i + blockSize/2] = temp;
|
Chris@67
|
272 }
|
Chris@67
|
273 fftw_execute(fftPlan);
|
Chris@67
|
274 for (size_t i = 0; i < blockSize/2; ++i) {
|
Chris@67
|
275 buffers[ch][i*2] = fftOutput[i][0];
|
Chris@67
|
276 buffers[ch][i*2 + 1] = fftOutput[i][1];
|
Chris@67
|
277 }
|
Chris@67
|
278 }
|
Chris@67
|
279 }
|
Chris@67
|
280
|
Chris@66
|
281 Vamp::Plugin::FeatureSet features = m_plugin->process
|
Chris@66
|
282 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
|
Chris@0
|
283
|
Chris@0
|
284 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@66
|
285 Vamp::Plugin::Feature feature =
|
Chris@0
|
286 features[m_outputFeatureNo][fi];
|
Chris@0
|
287 addFeature(blockFrame, feature);
|
Chris@0
|
288 }
|
Chris@0
|
289
|
Chris@0
|
290 if (blockFrame == startFrame || completion > prevCompletion) {
|
Chris@0
|
291 setCompletion(completion);
|
Chris@0
|
292 prevCompletion = completion;
|
Chris@0
|
293 }
|
Chris@0
|
294
|
Chris@0
|
295 blockFrame += stepSize;
|
Chris@0
|
296 }
|
Chris@0
|
297
|
Chris@67
|
298 if (fftPlan) {
|
Chris@67
|
299 fftw_destroy_plan(fftPlan);
|
Chris@67
|
300 fftw_free(fftInput);
|
Chris@67
|
301 fftw_free(fftOutput);
|
Chris@67
|
302 }
|
Chris@67
|
303
|
Chris@66
|
304 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
|
Chris@0
|
305
|
Chris@0
|
306 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@66
|
307 Vamp::Plugin::Feature feature =
|
Chris@0
|
308 features[m_outputFeatureNo][fi];
|
Chris@0
|
309 addFeature(blockFrame, feature);
|
Chris@0
|
310 }
|
Chris@0
|
311
|
Chris@0
|
312 setCompletion(100);
|
Chris@0
|
313 }
|
Chris@0
|
314
|
Chris@0
|
315
|
Chris@0
|
316 void
|
Chris@0
|
317 FeatureExtractionPluginTransform::addFeature(size_t blockFrame,
|
Chris@66
|
318 const Vamp::Plugin::Feature &feature)
|
Chris@0
|
319 {
|
Chris@0
|
320 size_t inputRate = m_input->getSampleRate();
|
Chris@0
|
321
|
Chris@0
|
322 // std::cerr << "FeatureExtractionPluginTransform::addFeature("
|
Chris@0
|
323 // << blockFrame << ")" << std::endl;
|
Chris@0
|
324
|
Chris@0
|
325 int valueCount = 1;
|
Chris@0
|
326 if (m_descriptor->hasFixedValueCount) {
|
Chris@0
|
327 valueCount = m_descriptor->valueCount;
|
Chris@0
|
328 }
|
Chris@0
|
329
|
Chris@0
|
330 size_t frame = blockFrame;
|
Chris@0
|
331
|
Chris@0
|
332 if (m_descriptor->sampleType ==
|
Chris@66
|
333 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
334
|
Chris@0
|
335 if (!feature.hasTimestamp) {
|
Chris@0
|
336 std::cerr
|
Chris@0
|
337 << "WARNING: FeatureExtractionPluginTransform::addFeature: "
|
Chris@0
|
338 << "Feature has variable sample rate but no timestamp!"
|
Chris@0
|
339 << std::endl;
|
Chris@0
|
340 return;
|
Chris@0
|
341 } else {
|
Chris@66
|
342 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
|
Chris@0
|
343 }
|
Chris@0
|
344
|
Chris@0
|
345 } else if (m_descriptor->sampleType ==
|
Chris@66
|
346 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
|
Chris@0
|
347
|
Chris@0
|
348 if (feature.hasTimestamp) {
|
Chris@0
|
349 //!!! warning: sampleRate may be non-integral
|
Chris@66
|
350 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
|
Chris@66
|
351 m_descriptor->sampleRate);
|
Chris@0
|
352 } else {
|
Chris@0
|
353 frame = m_output->getEndFrame() + 1;
|
Chris@0
|
354 }
|
Chris@0
|
355 }
|
Chris@0
|
356
|
Chris@0
|
357 if (valueCount == 0) {
|
Chris@0
|
358
|
Chris@0
|
359 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
|
Chris@0
|
360 if (!model) return;
|
Chris@0
|
361 model->addPoint(SparseOneDimensionalModel::Point(frame, feature.label.c_str()));
|
Chris@0
|
362
|
Chris@0
|
363 } else if (valueCount == 1 ||
|
Chris@0
|
364 m_descriptor->sampleType ==
|
Chris@66
|
365 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
366
|
Chris@0
|
367 float value = 0.0;
|
Chris@0
|
368 if (feature.values.size() > 0) value = feature.values[0];
|
Chris@0
|
369
|
Chris@0
|
370 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
|
Chris@0
|
371 if (!model) return;
|
Chris@0
|
372 model->addPoint(SparseTimeValueModel::Point(frame, value, feature.label.c_str()));
|
Chris@0
|
373
|
Chris@0
|
374 } else {
|
Chris@0
|
375
|
Chris@0
|
376 DenseThreeDimensionalModel::BinValueSet values = feature.values;
|
Chris@0
|
377
|
Chris@0
|
378 DenseThreeDimensionalModel *model = getOutput<DenseThreeDimensionalModel>();
|
Chris@0
|
379 if (!model) return;
|
Chris@0
|
380
|
Chris@0
|
381 model->setBinValues(frame, values);
|
Chris@0
|
382 }
|
Chris@0
|
383 }
|
Chris@0
|
384
|
Chris@0
|
385 void
|
Chris@0
|
386 FeatureExtractionPluginTransform::setCompletion(int completion)
|
Chris@0
|
387 {
|
Chris@0
|
388 int valueCount = 1;
|
Chris@0
|
389 if (m_descriptor->hasFixedValueCount) {
|
Chris@0
|
390 valueCount = m_descriptor->valueCount;
|
Chris@0
|
391 }
|
Chris@0
|
392
|
Chris@0
|
393 if (valueCount == 0) {
|
Chris@0
|
394
|
Chris@0
|
395 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
|
Chris@0
|
396 if (!model) return;
|
Chris@0
|
397 model->setCompletion(completion);
|
Chris@0
|
398
|
Chris@0
|
399 } else if (valueCount == 1 ||
|
Chris@0
|
400 m_descriptor->sampleType ==
|
Chris@66
|
401 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@0
|
402
|
Chris@0
|
403 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
|
Chris@0
|
404 if (!model) return;
|
Chris@0
|
405 model->setCompletion(completion);
|
Chris@0
|
406
|
Chris@0
|
407 } else {
|
Chris@0
|
408
|
Chris@19
|
409 DenseThreeDimensionalModel *model = getOutput<DenseThreeDimensionalModel>();
|
Chris@19
|
410 if (!model) return;
|
Chris@19
|
411 model->setCompletion(completion);
|
Chris@0
|
412 }
|
Chris@0
|
413 }
|
Chris@0
|
414
|