Chris@320
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@320
|
2
|
Chris@320
|
3 /*
|
Chris@320
|
4 Sonic Visualiser
|
Chris@320
|
5 An audio file viewer and annotation editor.
|
Chris@320
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@320
|
7 This file copyright 2006 Chris Cannam and QMUL.
|
Chris@320
|
8
|
Chris@320
|
9 This program is free software; you can redistribute it and/or
|
Chris@320
|
10 modify it under the terms of the GNU General Public License as
|
Chris@320
|
11 published by the Free Software Foundation; either version 2 of the
|
Chris@320
|
12 License, or (at your option) any later version. See the file
|
Chris@320
|
13 COPYING included with this distribution for more information.
|
Chris@320
|
14 */
|
Chris@320
|
15
|
Chris@331
|
16 #include "FeatureExtractionModelTransformer.h"
|
Chris@320
|
17
|
Chris@320
|
18 #include "plugin/FeatureExtractionPluginFactory.h"
|
Chris@320
|
19 #include "plugin/PluginXml.h"
|
Chris@320
|
20 #include "vamp-sdk/Plugin.h"
|
Chris@320
|
21
|
Chris@320
|
22 #include "data/model/Model.h"
|
Chris@320
|
23 #include "base/Window.h"
|
Chris@387
|
24 #include "base/Exceptions.h"
|
Chris@320
|
25 #include "data/model/SparseOneDimensionalModel.h"
|
Chris@320
|
26 #include "data/model/SparseTimeValueModel.h"
|
Chris@320
|
27 #include "data/model/EditableDenseThreeDimensionalModel.h"
|
Chris@320
|
28 #include "data/model/DenseTimeValueModel.h"
|
Chris@320
|
29 #include "data/model/NoteModel.h"
|
Chris@441
|
30 #include "data/model/RegionModel.h"
|
Chris@320
|
31 #include "data/model/FFTModel.h"
|
Chris@320
|
32 #include "data/model/WaveFileModel.h"
|
Chris@320
|
33
|
Chris@350
|
34 #include "TransformFactory.h"
|
Chris@350
|
35
|
Chris@320
|
36 #include <iostream>
|
Chris@320
|
37
|
Chris@350
|
38 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
|
Chris@350
|
39 const Transform &transform) :
|
Chris@350
|
40 ModelTransformer(in, transform),
|
Chris@320
|
41 m_plugin(0),
|
Chris@320
|
42 m_descriptor(0),
|
Chris@320
|
43 m_outputFeatureNo(0)
|
Chris@320
|
44 {
|
Chris@350
|
45 // std::cerr << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId.toStdString() << ", outputName " << m_transform.getOutput().toStdString() << std::endl;
|
Chris@350
|
46
|
Chris@350
|
47 QString pluginId = transform.getPluginIdentifier();
|
Chris@320
|
48
|
Chris@320
|
49 FeatureExtractionPluginFactory *factory =
|
Chris@320
|
50 FeatureExtractionPluginFactory::instanceFor(pluginId);
|
Chris@320
|
51
|
Chris@320
|
52 if (!factory) {
|
Chris@361
|
53 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
|
Chris@320
|
54 return;
|
Chris@320
|
55 }
|
Chris@320
|
56
|
Chris@350
|
57 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
58 if (!input) {
|
Chris@361
|
59 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
|
Chris@350
|
60 return;
|
Chris@350
|
61 }
|
Chris@320
|
62
|
Chris@350
|
63 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
|
Chris@320
|
64 if (!m_plugin) {
|
Chris@361
|
65 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
|
Chris@320
|
66 return;
|
Chris@320
|
67 }
|
Chris@320
|
68
|
Chris@350
|
69 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@350
|
70 (m_transform, m_plugin);
|
Chris@343
|
71
|
Chris@350
|
72 TransformFactory::getInstance()->setPluginParameters
|
Chris@350
|
73 (m_transform, m_plugin);
|
Chris@320
|
74
|
Chris@320
|
75 size_t channelCount = input->getChannelCount();
|
Chris@320
|
76 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
77 channelCount = 1;
|
Chris@320
|
78 }
|
Chris@320
|
79 if (m_plugin->getMinChannelCount() > channelCount) {
|
Chris@361
|
80 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
|
Chris@361
|
81 .arg(pluginId)
|
Chris@361
|
82 .arg(m_plugin->getMinChannelCount())
|
Chris@361
|
83 .arg(m_plugin->getMaxChannelCount())
|
Chris@361
|
84 .arg(input->getChannelCount());
|
Chris@320
|
85 return;
|
Chris@320
|
86 }
|
Chris@320
|
87
|
Chris@320
|
88 std::cerr << "Initialising feature extraction plugin with channels = "
|
Chris@350
|
89 << channelCount << ", step = " << m_transform.getStepSize()
|
Chris@350
|
90 << ", block = " << m_transform.getBlockSize() << std::endl;
|
Chris@320
|
91
|
Chris@320
|
92 if (!m_plugin->initialise(channelCount,
|
Chris@350
|
93 m_transform.getStepSize(),
|
Chris@350
|
94 m_transform.getBlockSize())) {
|
Chris@361
|
95
|
Chris@361
|
96 size_t pstep = m_transform.getStepSize();
|
Chris@361
|
97 size_t pblock = m_transform.getBlockSize();
|
Chris@361
|
98
|
Chris@361
|
99 m_transform.setStepSize(0);
|
Chris@361
|
100 m_transform.setBlockSize(0);
|
Chris@361
|
101 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@361
|
102 (m_transform, m_plugin);
|
Chris@361
|
103
|
Chris@361
|
104 if (m_transform.getStepSize() != pstep ||
|
Chris@361
|
105 m_transform.getBlockSize() != pblock) {
|
Chris@361
|
106
|
Chris@361
|
107 if (!m_plugin->initialise(channelCount,
|
Chris@361
|
108 m_transform.getStepSize(),
|
Chris@361
|
109 m_transform.getBlockSize())) {
|
Chris@361
|
110
|
Chris@361
|
111 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
112 return;
|
Chris@361
|
113
|
Chris@361
|
114 } else {
|
Chris@361
|
115
|
Chris@361
|
116 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
|
Chris@361
|
117 .arg(pluginId)
|
Chris@361
|
118 .arg(pstep)
|
Chris@361
|
119 .arg(pblock)
|
Chris@361
|
120 .arg(m_transform.getStepSize())
|
Chris@361
|
121 .arg(m_transform.getBlockSize());
|
Chris@361
|
122 }
|
Chris@361
|
123
|
Chris@361
|
124 } else {
|
Chris@361
|
125
|
Chris@361
|
126 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
127 return;
|
Chris@361
|
128 }
|
Chris@320
|
129 }
|
Chris@320
|
130
|
Chris@366
|
131 if (m_transform.getPluginVersion() != "") {
|
Chris@366
|
132 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
|
Chris@366
|
133 if (pv != m_transform.getPluginVersion()) {
|
Chris@366
|
134 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
|
Chris@366
|
135 .arg(m_transform.getPluginVersion())
|
Chris@366
|
136 .arg(pluginId)
|
Chris@366
|
137 .arg(pv);
|
Chris@366
|
138 if (m_message != "") {
|
Chris@366
|
139 m_message = QString("%1; %2").arg(vm).arg(m_message);
|
Chris@366
|
140 } else {
|
Chris@366
|
141 m_message = vm;
|
Chris@366
|
142 }
|
Chris@366
|
143 }
|
Chris@366
|
144 }
|
Chris@366
|
145
|
Chris@320
|
146 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
|
Chris@320
|
147
|
Chris@320
|
148 if (outputs.empty()) {
|
Chris@361
|
149 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
|
Chris@320
|
150 return;
|
Chris@320
|
151 }
|
Chris@320
|
152
|
Chris@320
|
153 for (size_t i = 0; i < outputs.size(); ++i) {
|
Chris@429
|
154 // std::cerr << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput().toStdString() << "\"" << std::endl;
|
Chris@350
|
155 if (m_transform.getOutput() == "" ||
|
Chris@350
|
156 outputs[i].identifier == m_transform.getOutput().toStdString()) {
|
Chris@320
|
157 m_outputFeatureNo = i;
|
Chris@441
|
158 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
|
Chris@320
|
159 break;
|
Chris@320
|
160 }
|
Chris@320
|
161 }
|
Chris@320
|
162
|
Chris@320
|
163 if (!m_descriptor) {
|
Chris@361
|
164 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
|
Chris@361
|
165 .arg(pluginId)
|
Chris@361
|
166 .arg(m_transform.getOutput());
|
Chris@320
|
167 return;
|
Chris@320
|
168 }
|
Chris@320
|
169
|
Chris@331
|
170 // std::cerr << "FeatureExtractionModelTransformer: output sample type "
|
Chris@320
|
171 // << m_descriptor->sampleType << std::endl;
|
Chris@320
|
172
|
Chris@320
|
173 int binCount = 1;
|
Chris@320
|
174 float minValue = 0.0, maxValue = 0.0;
|
Chris@320
|
175 bool haveExtents = false;
|
Chris@320
|
176
|
Chris@320
|
177 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
178 binCount = m_descriptor->binCount;
|
Chris@320
|
179 }
|
Chris@320
|
180
|
Chris@331
|
181 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
|
Chris@320
|
182 // << binCount << std::endl;
|
Chris@320
|
183
|
Chris@320
|
184 if (binCount > 0 && m_descriptor->hasKnownExtents) {
|
Chris@320
|
185 minValue = m_descriptor->minValue;
|
Chris@320
|
186 maxValue = m_descriptor->maxValue;
|
Chris@320
|
187 haveExtents = true;
|
Chris@320
|
188 }
|
Chris@320
|
189
|
Chris@350
|
190 size_t modelRate = input->getSampleRate();
|
Chris@320
|
191 size_t modelResolution = 1;
|
Chris@320
|
192
|
Chris@320
|
193 switch (m_descriptor->sampleType) {
|
Chris@320
|
194
|
Chris@320
|
195 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
|
Chris@320
|
196 if (m_descriptor->sampleRate != 0.0) {
|
Chris@320
|
197 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
|
Chris@320
|
198 }
|
Chris@320
|
199 break;
|
Chris@320
|
200
|
Chris@320
|
201 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
|
Chris@350
|
202 modelResolution = m_transform.getStepSize();
|
Chris@320
|
203 break;
|
Chris@320
|
204
|
Chris@320
|
205 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
|
Chris@451
|
206 //!!! SV doesn't actually support display of models that have
|
Chris@451
|
207 //!!! different underlying rates together -- so we always set
|
Chris@451
|
208 //!!! the model rate to be the input model's rate, and adjust
|
Chris@451
|
209 //!!! the resolution appropriately. We can't properly display
|
Chris@451
|
210 //!!! data with a higher resolution than the base model at all
|
Chris@451
|
211 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
|
Chris@451
|
212 if (m_descriptor->sampleRate > input->getSampleRate()) {
|
Chris@451
|
213 modelResolution = 1;
|
Chris@451
|
214 } else {
|
Chris@451
|
215 modelResolution = size_t(input->getSampleRate() /
|
Chris@451
|
216 m_descriptor->sampleRate);
|
Chris@451
|
217 }
|
Chris@320
|
218 break;
|
Chris@320
|
219 }
|
Chris@320
|
220
|
Chris@441
|
221 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
|
Chris@441
|
222
|
Chris@441
|
223 if (binCount == 0 &&
|
Chris@441
|
224 (preDurationPlugin || !m_descriptor->hasDuration)) {
|
Chris@320
|
225
|
Chris@445
|
226 // Anything with no value and no duration is an instant
|
Chris@445
|
227
|
Chris@320
|
228 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
|
Chris@320
|
229 false);
|
Chris@320
|
230
|
Chris@441
|
231 } else if ((preDurationPlugin && binCount > 1 &&
|
Chris@441
|
232 (m_descriptor->sampleType ==
|
Chris@441
|
233 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
|
Chris@441
|
234 (!preDurationPlugin && m_descriptor->hasDuration)) {
|
Chris@441
|
235
|
Chris@441
|
236 // For plugins using the old v1 API without explicit duration,
|
Chris@441
|
237 // we treat anything that has multiple bins (i.e. that has the
|
Chris@441
|
238 // potential to have value and duration) and a variable sample
|
Chris@441
|
239 // rate as a note model, taking its values as pitch, duration
|
Chris@441
|
240 // and velocity (if present) respectively. This is the same
|
Chris@441
|
241 // behaviour as always applied by SV to these plugins in the
|
Chris@441
|
242 // past.
|
Chris@441
|
243
|
Chris@441
|
244 // For plugins with the newer API, we treat anything with
|
Chris@441
|
245 // duration as either a note model with pitch and velocity, or
|
Chris@441
|
246 // a region model.
|
Chris@441
|
247
|
Chris@441
|
248 // How do we know whether it's an interval or note model?
|
Chris@441
|
249 // What's the essential difference? Is a note model any
|
Chris@441
|
250 // interval model using a Hz or "MIDI pitch" scale? There
|
Chris@441
|
251 // isn't really a reliable test for "MIDI pitch"... Does a
|
Chris@441
|
252 // note model always have velocity? This is a good question
|
Chris@441
|
253 // to be addressed by accompanying RDF, but for the moment we
|
Chris@441
|
254 // will do the following...
|
Chris@441
|
255
|
Chris@441
|
256 bool isNoteModel = false;
|
Chris@441
|
257
|
Chris@441
|
258 // Regions have only value (and duration -- we can't extract a
|
Chris@441
|
259 // region model from an old-style plugin that doesn't support
|
Chris@441
|
260 // duration)
|
Chris@441
|
261 if (binCount > 1) isNoteModel = true;
|
Chris@441
|
262
|
Chris@441
|
263 // Regions do not have units of Hz (a sweeping assumption!)
|
Chris@441
|
264 if (m_descriptor->unit == "Hz") isNoteModel = true;
|
Chris@441
|
265
|
Chris@441
|
266 // If we had a "sparse 3D model", we would have the additional
|
Chris@441
|
267 // problem of determining whether to use that here (if bin
|
Chris@441
|
268 // count > 1). But we don't.
|
Chris@441
|
269
|
Chris@441
|
270 if (isNoteModel) {
|
Chris@441
|
271
|
Chris@441
|
272 NoteModel *model;
|
Chris@441
|
273 if (haveExtents) {
|
Chris@441
|
274 model = new NoteModel
|
Chris@441
|
275 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
276 } else {
|
Chris@441
|
277 model = new NoteModel
|
Chris@441
|
278 (modelRate, modelResolution, false);
|
Chris@441
|
279 }
|
Chris@441
|
280 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
281 m_output = model;
|
Chris@441
|
282
|
Chris@441
|
283 } else {
|
Chris@441
|
284
|
Chris@441
|
285 RegionModel *model;
|
Chris@441
|
286 if (haveExtents) {
|
Chris@441
|
287 model = new RegionModel
|
Chris@441
|
288 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
289 } else {
|
Chris@441
|
290 model = new RegionModel
|
Chris@441
|
291 (modelRate, modelResolution, false);
|
Chris@441
|
292 }
|
Chris@441
|
293 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
294 m_output = model;
|
Chris@441
|
295 }
|
Chris@441
|
296
|
Chris@441
|
297 } else if (binCount == 1 ||
|
Chris@441
|
298 (m_descriptor->sampleType ==
|
Chris@441
|
299 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
|
Chris@441
|
300
|
Chris@441
|
301 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
302 // has only one value per result must be a sparse time value
|
Chris@441
|
303 // model.
|
Chris@441
|
304
|
Chris@441
|
305 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
306 // has a variable sample rate is also treated as a sparse time
|
Chris@441
|
307 // value model regardless of its bin count, because we lack a
|
Chris@441
|
308 // sparse 3D model.
|
Chris@320
|
309
|
Chris@320
|
310 SparseTimeValueModel *model;
|
Chris@320
|
311 if (haveExtents) {
|
Chris@320
|
312 model = new SparseTimeValueModel
|
Chris@320
|
313 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@320
|
314 } else {
|
Chris@320
|
315 model = new SparseTimeValueModel
|
Chris@320
|
316 (modelRate, modelResolution, false);
|
Chris@320
|
317 }
|
Chris@320
|
318 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
|
Chris@320
|
319
|
Chris@320
|
320 m_output = model;
|
Chris@320
|
321
|
Chris@441
|
322 } else {
|
Chris@320
|
323
|
Chris@441
|
324 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
325 // has a fixed sample rate and more than one value per result
|
Chris@441
|
326 // must be a dense 3D model.
|
Chris@320
|
327
|
Chris@320
|
328 EditableDenseThreeDimensionalModel *model =
|
Chris@320
|
329 new EditableDenseThreeDimensionalModel
|
Chris@320
|
330 (modelRate, modelResolution, binCount, false);
|
Chris@320
|
331
|
Chris@320
|
332 if (!m_descriptor->binNames.empty()) {
|
Chris@320
|
333 std::vector<QString> names;
|
Chris@320
|
334 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
|
Chris@320
|
335 names.push_back(m_descriptor->binNames[i].c_str());
|
Chris@320
|
336 }
|
Chris@320
|
337 model->setBinNames(names);
|
Chris@320
|
338 }
|
Chris@320
|
339
|
Chris@320
|
340 m_output = model;
|
Chris@320
|
341 }
|
Chris@333
|
342
|
Chris@350
|
343 if (m_output) m_output->setSourceModel(input);
|
Chris@320
|
344 }
|
Chris@320
|
345
|
Chris@331
|
346 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
|
Chris@320
|
347 {
|
Chris@436
|
348 // std::cerr << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << std::endl;
|
Chris@320
|
349 delete m_plugin;
|
Chris@320
|
350 delete m_descriptor;
|
Chris@320
|
351 }
|
Chris@320
|
352
|
Chris@320
|
353 DenseTimeValueModel *
|
Chris@350
|
354 FeatureExtractionModelTransformer::getConformingInput()
|
Chris@320
|
355 {
|
Chris@408
|
356 // std::cerr << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << std::endl;
|
Chris@408
|
357
|
Chris@320
|
358 DenseTimeValueModel *dtvm =
|
Chris@320
|
359 dynamic_cast<DenseTimeValueModel *>(getInputModel());
|
Chris@320
|
360 if (!dtvm) {
|
Chris@350
|
361 std::cerr << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
|
Chris@320
|
362 }
|
Chris@320
|
363 return dtvm;
|
Chris@320
|
364 }
|
Chris@320
|
365
|
Chris@320
|
366 void
|
Chris@331
|
367 FeatureExtractionModelTransformer::run()
|
Chris@320
|
368 {
|
Chris@350
|
369 DenseTimeValueModel *input = getConformingInput();
|
Chris@320
|
370 if (!input) return;
|
Chris@320
|
371
|
Chris@320
|
372 if (!m_output) return;
|
Chris@320
|
373
|
Chris@320
|
374 while (!input->isReady()) {
|
Chris@331
|
375 std::cerr << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << std::endl;
|
Chris@320
|
376 sleep(1);
|
Chris@320
|
377 }
|
Chris@320
|
378
|
Chris@350
|
379 size_t sampleRate = input->getSampleRate();
|
Chris@320
|
380
|
Chris@320
|
381 size_t channelCount = input->getChannelCount();
|
Chris@320
|
382 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
383 channelCount = 1;
|
Chris@320
|
384 }
|
Chris@320
|
385
|
Chris@320
|
386 float **buffers = new float*[channelCount];
|
Chris@320
|
387 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
388 buffers[ch] = new float[m_transform.getBlockSize() + 2];
|
Chris@320
|
389 }
|
Chris@320
|
390
|
Chris@350
|
391 size_t stepSize = m_transform.getStepSize();
|
Chris@350
|
392 size_t blockSize = m_transform.getBlockSize();
|
Chris@350
|
393
|
Chris@320
|
394 bool frequencyDomain = (m_plugin->getInputDomain() ==
|
Chris@320
|
395 Vamp::Plugin::FrequencyDomain);
|
Chris@320
|
396 std::vector<FFTModel *> fftModels;
|
Chris@320
|
397
|
Chris@320
|
398 if (frequencyDomain) {
|
Chris@320
|
399 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
400 FFTModel *model = new FFTModel
|
Chris@350
|
401 (getConformingInput(),
|
Chris@350
|
402 channelCount == 1 ? m_input.getChannel() : ch,
|
Chris@350
|
403 m_transform.getWindowType(),
|
Chris@350
|
404 blockSize,
|
Chris@350
|
405 stepSize,
|
Chris@350
|
406 blockSize,
|
Chris@334
|
407 false,
|
Chris@334
|
408 StorageAdviser::PrecisionCritical);
|
Chris@320
|
409 if (!model->isOK()) {
|
Chris@320
|
410 delete model;
|
Chris@320
|
411 setCompletion(100);
|
Chris@387
|
412 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
|
Chris@387
|
413 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
|
Chris@320
|
414 }
|
Chris@320
|
415 model->resume();
|
Chris@320
|
416 fftModels.push_back(model);
|
Chris@320
|
417 }
|
Chris@320
|
418 }
|
Chris@320
|
419
|
Chris@350
|
420 long startFrame = m_input.getModel()->getStartFrame();
|
Chris@350
|
421 long endFrame = m_input.getModel()->getEndFrame();
|
Chris@320
|
422
|
Chris@350
|
423 RealTime contextStartRT = m_transform.getStartTime();
|
Chris@350
|
424 RealTime contextDurationRT = m_transform.getDuration();
|
Chris@350
|
425
|
Chris@350
|
426 long contextStart =
|
Chris@350
|
427 RealTime::realTime2Frame(contextStartRT, sampleRate);
|
Chris@350
|
428
|
Chris@350
|
429 long contextDuration =
|
Chris@350
|
430 RealTime::realTime2Frame(contextDurationRT, sampleRate);
|
Chris@320
|
431
|
Chris@320
|
432 if (contextStart == 0 || contextStart < startFrame) {
|
Chris@320
|
433 contextStart = startFrame;
|
Chris@320
|
434 }
|
Chris@320
|
435
|
Chris@320
|
436 if (contextDuration == 0) {
|
Chris@320
|
437 contextDuration = endFrame - contextStart;
|
Chris@320
|
438 }
|
Chris@320
|
439 if (contextStart + contextDuration > endFrame) {
|
Chris@320
|
440 contextDuration = endFrame - contextStart;
|
Chris@320
|
441 }
|
Chris@320
|
442
|
Chris@320
|
443 long blockFrame = contextStart;
|
Chris@320
|
444
|
Chris@320
|
445 long prevCompletion = 0;
|
Chris@320
|
446
|
Chris@320
|
447 setCompletion(0);
|
Chris@320
|
448
|
Chris@320
|
449 while (!m_abandoned) {
|
Chris@320
|
450
|
Chris@320
|
451 if (frequencyDomain) {
|
Chris@350
|
452 if (blockFrame - int(blockSize)/2 >
|
Chris@320
|
453 contextStart + contextDuration) break;
|
Chris@320
|
454 } else {
|
Chris@320
|
455 if (blockFrame >=
|
Chris@320
|
456 contextStart + contextDuration) break;
|
Chris@320
|
457 }
|
Chris@320
|
458
|
Chris@331
|
459 // std::cerr << "FeatureExtractionModelTransformer::run: blockFrame "
|
Chris@320
|
460 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
|
Chris@350
|
461 // << blockSize << std::endl;
|
Chris@320
|
462
|
Chris@320
|
463 long completion =
|
Chris@350
|
464 (((blockFrame - contextStart) / stepSize) * 99) /
|
Chris@350
|
465 (contextDuration / stepSize);
|
Chris@320
|
466
|
Chris@350
|
467 // channelCount is either m_input.getModel()->channelCount or 1
|
Chris@320
|
468
|
Chris@363
|
469 if (frequencyDomain) {
|
Chris@363
|
470 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
471 int column = (blockFrame - startFrame) / stepSize;
|
Chris@350
|
472 for (size_t i = 0; i <= blockSize/2; ++i) {
|
Chris@320
|
473 fftModels[ch]->getValuesAt
|
Chris@320
|
474 (column, i, buffers[ch][i*2], buffers[ch][i*2+1]);
|
Chris@320
|
475 }
|
Chris@363
|
476 }
|
Chris@363
|
477 } else {
|
Chris@363
|
478 getFrames(channelCount, blockFrame, blockSize, buffers);
|
Chris@320
|
479 }
|
Chris@320
|
480
|
Chris@320
|
481 Vamp::Plugin::FeatureSet features = m_plugin->process
|
Chris@320
|
482 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
|
Chris@320
|
483
|
Chris@320
|
484 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@320
|
485 Vamp::Plugin::Feature feature =
|
Chris@320
|
486 features[m_outputFeatureNo][fi];
|
Chris@320
|
487 addFeature(blockFrame, feature);
|
Chris@320
|
488 }
|
Chris@320
|
489
|
Chris@320
|
490 if (blockFrame == contextStart || completion > prevCompletion) {
|
Chris@320
|
491 setCompletion(completion);
|
Chris@320
|
492 prevCompletion = completion;
|
Chris@320
|
493 }
|
Chris@320
|
494
|
Chris@350
|
495 blockFrame += stepSize;
|
Chris@320
|
496 }
|
Chris@320
|
497
|
Chris@320
|
498 if (m_abandoned) return;
|
Chris@320
|
499
|
Chris@320
|
500 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
|
Chris@320
|
501
|
Chris@320
|
502 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@320
|
503 Vamp::Plugin::Feature feature =
|
Chris@320
|
504 features[m_outputFeatureNo][fi];
|
Chris@320
|
505 addFeature(blockFrame, feature);
|
Chris@320
|
506 }
|
Chris@320
|
507
|
Chris@320
|
508 if (frequencyDomain) {
|
Chris@320
|
509 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
510 delete fftModels[ch];
|
Chris@320
|
511 }
|
Chris@320
|
512 }
|
Chris@320
|
513
|
Chris@320
|
514 setCompletion(100);
|
Chris@320
|
515 }
|
Chris@320
|
516
|
Chris@320
|
517 void
|
Chris@363
|
518 FeatureExtractionModelTransformer::getFrames(int channelCount,
|
Chris@363
|
519 long startFrame, long size,
|
Chris@363
|
520 float **buffers)
|
Chris@320
|
521 {
|
Chris@320
|
522 long offset = 0;
|
Chris@320
|
523
|
Chris@320
|
524 if (startFrame < 0) {
|
Chris@363
|
525 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
526 for (int i = 0; i < size && startFrame + i < 0; ++i) {
|
Chris@363
|
527 buffers[c][i] = 0.0f;
|
Chris@363
|
528 }
|
Chris@320
|
529 }
|
Chris@320
|
530 offset = -startFrame;
|
Chris@320
|
531 size -= offset;
|
Chris@320
|
532 if (size <= 0) return;
|
Chris@320
|
533 startFrame = 0;
|
Chris@320
|
534 }
|
Chris@320
|
535
|
Chris@350
|
536 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
537 if (!input) return;
|
Chris@363
|
538
|
Chris@363
|
539 long got = 0;
|
Chris@350
|
540
|
Chris@363
|
541 if (channelCount == 1) {
|
Chris@363
|
542
|
Chris@363
|
543 got = input->getData(m_input.getChannel(), startFrame, size,
|
Chris@363
|
544 buffers[0] + offset);
|
Chris@363
|
545
|
Chris@363
|
546 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
|
Chris@363
|
547 // use mean instead of sum, as plugin input
|
Chris@363
|
548 float cc = float(input->getChannelCount());
|
Chris@363
|
549 for (long i = 0; i < size; ++i) {
|
Chris@363
|
550 buffers[0][i + offset] /= cc;
|
Chris@363
|
551 }
|
Chris@363
|
552 }
|
Chris@363
|
553
|
Chris@363
|
554 } else {
|
Chris@363
|
555
|
Chris@363
|
556 float **writebuf = buffers;
|
Chris@363
|
557 if (offset > 0) {
|
Chris@363
|
558 writebuf = new float *[channelCount];
|
Chris@363
|
559 for (int i = 0; i < channelCount; ++i) {
|
Chris@363
|
560 writebuf[i] = buffers[i] + offset;
|
Chris@363
|
561 }
|
Chris@363
|
562 }
|
Chris@363
|
563
|
Chris@363
|
564 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
|
Chris@363
|
565
|
Chris@363
|
566 if (writebuf != buffers) delete[] writebuf;
|
Chris@363
|
567 }
|
Chris@320
|
568
|
Chris@320
|
569 while (got < size) {
|
Chris@363
|
570 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
571 buffers[c][got + offset] = 0.0;
|
Chris@363
|
572 }
|
Chris@320
|
573 ++got;
|
Chris@320
|
574 }
|
Chris@320
|
575 }
|
Chris@320
|
576
|
Chris@320
|
577 void
|
Chris@331
|
578 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
|
Chris@320
|
579 const Vamp::Plugin::Feature &feature)
|
Chris@320
|
580 {
|
Chris@350
|
581 size_t inputRate = m_input.getModel()->getSampleRate();
|
Chris@320
|
582
|
Chris@331
|
583 // std::cerr << "FeatureExtractionModelTransformer::addFeature("
|
Chris@320
|
584 // << blockFrame << ")" << std::endl;
|
Chris@320
|
585
|
Chris@320
|
586 int binCount = 1;
|
Chris@320
|
587 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
588 binCount = m_descriptor->binCount;
|
Chris@320
|
589 }
|
Chris@320
|
590
|
Chris@320
|
591 size_t frame = blockFrame;
|
Chris@320
|
592
|
Chris@320
|
593 if (m_descriptor->sampleType ==
|
Chris@320
|
594 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@320
|
595
|
Chris@320
|
596 if (!feature.hasTimestamp) {
|
Chris@320
|
597 std::cerr
|
Chris@331
|
598 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
|
Chris@320
|
599 << "Feature has variable sample rate but no timestamp!"
|
Chris@320
|
600 << std::endl;
|
Chris@320
|
601 return;
|
Chris@320
|
602 } else {
|
Chris@320
|
603 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
|
Chris@320
|
604 }
|
Chris@320
|
605
|
Chris@320
|
606 } else if (m_descriptor->sampleType ==
|
Chris@320
|
607 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
|
Chris@320
|
608
|
Chris@320
|
609 if (feature.hasTimestamp) {
|
Chris@320
|
610 //!!! warning: sampleRate may be non-integral
|
Chris@320
|
611 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
|
Chris@451
|
612 //!!! see comment above when setting up modelResolution and modelRate
|
Chris@451
|
613 // lrintf(m_descriptor->sampleRate));
|
Chris@451
|
614 inputRate);
|
Chris@320
|
615 } else {
|
Chris@320
|
616 frame = m_output->getEndFrame();
|
Chris@320
|
617 }
|
Chris@320
|
618 }
|
Chris@320
|
619
|
Chris@441
|
620 // Rather than repeat the complicated tests from the constructor
|
Chris@441
|
621 // to determine what sort of model we must be adding the features
|
Chris@441
|
622 // to, we instead test what sort of model the constructor decided
|
Chris@441
|
623 // to create.
|
Chris@320
|
624
|
Chris@441
|
625 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@441
|
626
|
Chris@441
|
627 SparseOneDimensionalModel *model =
|
Chris@350
|
628 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
629 if (!model) return;
|
Chris@350
|
630
|
Chris@441
|
631 model->addPoint(SparseOneDimensionalModel::Point
|
Chris@441
|
632 (frame, feature.label.c_str()));
|
Chris@320
|
633
|
Chris@441
|
634 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
635
|
Chris@350
|
636 SparseTimeValueModel *model =
|
Chris@350
|
637 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
638 if (!model) return;
|
Chris@350
|
639
|
Chris@454
|
640 for (int i = 0; i < feature.values.size(); ++i) {
|
Chris@454
|
641
|
Chris@454
|
642 float value = feature.values[i];
|
Chris@454
|
643
|
Chris@454
|
644 QString label = feature.label.c_str();
|
Chris@454
|
645 if (feature.values.size() > 1) {
|
Chris@454
|
646 label = QString("[%1] %2").arg(i+1).arg(label);
|
Chris@454
|
647 }
|
Chris@454
|
648
|
Chris@454
|
649 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
|
Chris@454
|
650 }
|
Chris@320
|
651
|
Chris@441
|
652 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
|
Chris@320
|
653
|
Chris@441
|
654 int index = 0;
|
Chris@441
|
655
|
Chris@441
|
656 float value = 0.0;
|
Chris@441
|
657 if (feature.values.size() > index) {
|
Chris@441
|
658 value = feature.values[index++];
|
Chris@441
|
659 }
|
Chris@320
|
660
|
Chris@320
|
661 float duration = 1;
|
Chris@441
|
662 if (feature.hasDuration) {
|
Chris@441
|
663 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
|
Chris@441
|
664 } else {
|
Chris@441
|
665 if (feature.values.size() > index) {
|
Chris@441
|
666 duration = feature.values[index++];
|
Chris@441
|
667 }
|
Chris@441
|
668 }
|
Chris@320
|
669
|
Chris@441
|
670 if (isOutput<NoteModel>()) {
|
Chris@320
|
671
|
Chris@441
|
672 float velocity = 100;
|
Chris@441
|
673 if (feature.values.size() > index) {
|
Chris@441
|
674 velocity = feature.values[index++];
|
Chris@441
|
675 }
|
Chris@441
|
676 if (velocity < 0) velocity = 127;
|
Chris@441
|
677 if (velocity > 127) velocity = 127;
|
Chris@320
|
678
|
Chris@441
|
679 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@441
|
680 if (!model) return;
|
Chris@441
|
681 model->addPoint(NoteModel::Point(frame, value, // value is pitch
|
Chris@441
|
682 lrintf(duration),
|
Chris@441
|
683 velocity / 127.f,
|
Chris@441
|
684 feature.label.c_str()));
|
Chris@441
|
685 } else {
|
Chris@441
|
686 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@454
|
687 if (!model) return;
|
Chris@454
|
688
|
Chris@454
|
689 if (feature.hasDuration) {
|
Chris@454
|
690
|
Chris@454
|
691 for (int i = 0; i < feature.values.size(); ++i) {
|
Chris@454
|
692
|
Chris@454
|
693 float value = feature.values[i];
|
Chris@454
|
694
|
Chris@454
|
695 QString label = feature.label.c_str();
|
Chris@454
|
696 if (feature.values.size() > 1) {
|
Chris@454
|
697 label = QString("[%1] %2").arg(i+1).arg(label);
|
Chris@454
|
698 }
|
Chris@454
|
699
|
Chris@454
|
700 model->addPoint(RegionModel::Point(frame, value,
|
Chris@454
|
701 lrintf(duration),
|
Chris@454
|
702 label));
|
Chris@454
|
703 }
|
Chris@454
|
704 } else {
|
Chris@454
|
705
|
Chris@441
|
706 model->addPoint(RegionModel::Point(frame, value,
|
Chris@441
|
707 lrintf(duration),
|
Chris@441
|
708 feature.label.c_str()));
|
Chris@454
|
709 }
|
Chris@441
|
710 }
|
Chris@320
|
711
|
Chris@441
|
712 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
713
|
Chris@320
|
714 DenseThreeDimensionalModel::Column values = feature.values;
|
Chris@320
|
715
|
Chris@320
|
716 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
717 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
718 if (!model) return;
|
Chris@320
|
719
|
Chris@320
|
720 model->setColumn(frame / model->getResolution(), values);
|
Chris@441
|
721
|
Chris@441
|
722 } else {
|
Chris@441
|
723 std::cerr << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << std::endl;
|
Chris@320
|
724 }
|
Chris@320
|
725 }
|
Chris@320
|
726
|
Chris@320
|
727 void
|
Chris@331
|
728 FeatureExtractionModelTransformer::setCompletion(int completion)
|
Chris@320
|
729 {
|
Chris@320
|
730 int binCount = 1;
|
Chris@320
|
731 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
732 binCount = m_descriptor->binCount;
|
Chris@320
|
733 }
|
Chris@320
|
734
|
Chris@331
|
735 // std::cerr << "FeatureExtractionModelTransformer::setCompletion("
|
Chris@320
|
736 // << completion << ")" << std::endl;
|
Chris@320
|
737
|
Chris@441
|
738 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@320
|
739
|
Chris@350
|
740 SparseOneDimensionalModel *model =
|
Chris@350
|
741 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
742 if (!model) return;
|
Chris@441
|
743 model->setCompletion(completion, true);
|
Chris@320
|
744
|
Chris@441
|
745 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
746
|
Chris@350
|
747 SparseTimeValueModel *model =
|
Chris@350
|
748 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
749 if (!model) return;
|
Chris@441
|
750 model->setCompletion(completion, true);
|
Chris@320
|
751
|
Chris@441
|
752 } else if (isOutput<NoteModel>()) {
|
Chris@320
|
753
|
Chris@441
|
754 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@320
|
755 if (!model) return;
|
Chris@441
|
756 model->setCompletion(completion, true);
|
Chris@320
|
757
|
Chris@441
|
758 } else if (isOutput<RegionModel>()) {
|
Chris@441
|
759
|
Chris@441
|
760 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@441
|
761 if (!model) return;
|
Chris@441
|
762 model->setCompletion(completion, true);
|
Chris@441
|
763
|
Chris@441
|
764 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
765
|
Chris@320
|
766 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
767 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
768 if (!model) return;
|
Chris@350
|
769 model->setCompletion(completion, true); //!!!m_context.updates);
|
Chris@320
|
770 }
|
Chris@320
|
771 }
|
Chris@320
|
772
|