Chris@320
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@320
|
2
|
Chris@320
|
3 /*
|
Chris@320
|
4 Sonic Visualiser
|
Chris@320
|
5 An audio file viewer and annotation editor.
|
Chris@320
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@320
|
7 This file copyright 2006 Chris Cannam and QMUL.
|
Chris@320
|
8
|
Chris@320
|
9 This program is free software; you can redistribute it and/or
|
Chris@320
|
10 modify it under the terms of the GNU General Public License as
|
Chris@320
|
11 published by the Free Software Foundation; either version 2 of the
|
Chris@320
|
12 License, or (at your option) any later version. See the file
|
Chris@320
|
13 COPYING included with this distribution for more information.
|
Chris@320
|
14 */
|
Chris@320
|
15
|
Chris@331
|
16 #include "FeatureExtractionModelTransformer.h"
|
Chris@320
|
17
|
Chris@320
|
18 #include "plugin/FeatureExtractionPluginFactory.h"
|
Chris@320
|
19 #include "plugin/PluginXml.h"
|
Chris@475
|
20 #include <vamp-hostsdk/Plugin.h>
|
Chris@320
|
21
|
Chris@320
|
22 #include "data/model/Model.h"
|
Chris@320
|
23 #include "base/Window.h"
|
Chris@387
|
24 #include "base/Exceptions.h"
|
Chris@320
|
25 #include "data/model/SparseOneDimensionalModel.h"
|
Chris@320
|
26 #include "data/model/SparseTimeValueModel.h"
|
Chris@320
|
27 #include "data/model/EditableDenseThreeDimensionalModel.h"
|
Chris@320
|
28 #include "data/model/DenseTimeValueModel.h"
|
Chris@320
|
29 #include "data/model/NoteModel.h"
|
Chris@441
|
30 #include "data/model/RegionModel.h"
|
Chris@320
|
31 #include "data/model/FFTModel.h"
|
Chris@320
|
32 #include "data/model/WaveFileModel.h"
|
Chris@558
|
33 #include "rdf/PluginRDFDescription.h"
|
Chris@320
|
34
|
Chris@350
|
35 #include "TransformFactory.h"
|
Chris@350
|
36
|
Chris@320
|
37 #include <iostream>
|
Chris@320
|
38
|
Chris@350
|
39 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
|
Chris@350
|
40 const Transform &transform) :
|
Chris@350
|
41 ModelTransformer(in, transform),
|
Chris@320
|
42 m_plugin(0),
|
Chris@320
|
43 m_descriptor(0),
|
Chris@320
|
44 m_outputFeatureNo(0)
|
Chris@320
|
45 {
|
Chris@687
|
46 // DEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId << ", outputName " << m_transform.getOutput() << endl;
|
Chris@350
|
47
|
Chris@350
|
48 QString pluginId = transform.getPluginIdentifier();
|
Chris@320
|
49
|
Chris@320
|
50 FeatureExtractionPluginFactory *factory =
|
Chris@320
|
51 FeatureExtractionPluginFactory::instanceFor(pluginId);
|
Chris@320
|
52
|
Chris@320
|
53 if (!factory) {
|
Chris@361
|
54 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
|
Chris@320
|
55 return;
|
Chris@320
|
56 }
|
Chris@320
|
57
|
Chris@350
|
58 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
59 if (!input) {
|
Chris@361
|
60 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
|
Chris@350
|
61 return;
|
Chris@350
|
62 }
|
Chris@320
|
63
|
Chris@350
|
64 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
|
Chris@320
|
65 if (!m_plugin) {
|
Chris@361
|
66 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
|
Chris@320
|
67 return;
|
Chris@320
|
68 }
|
Chris@320
|
69
|
Chris@350
|
70 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@350
|
71 (m_transform, m_plugin);
|
Chris@343
|
72
|
Chris@350
|
73 TransformFactory::getInstance()->setPluginParameters
|
Chris@350
|
74 (m_transform, m_plugin);
|
Chris@320
|
75
|
Chris@320
|
76 size_t channelCount = input->getChannelCount();
|
Chris@320
|
77 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
78 channelCount = 1;
|
Chris@320
|
79 }
|
Chris@320
|
80 if (m_plugin->getMinChannelCount() > channelCount) {
|
Chris@361
|
81 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
|
Chris@361
|
82 .arg(pluginId)
|
Chris@361
|
83 .arg(m_plugin->getMinChannelCount())
|
Chris@361
|
84 .arg(m_plugin->getMaxChannelCount())
|
Chris@361
|
85 .arg(input->getChannelCount());
|
Chris@320
|
86 return;
|
Chris@320
|
87 }
|
Chris@320
|
88
|
Chris@687
|
89 DEBUG << "Initialising feature extraction plugin with channels = "
|
Chris@350
|
90 << channelCount << ", step = " << m_transform.getStepSize()
|
Chris@687
|
91 << ", block = " << m_transform.getBlockSize() << endl;
|
Chris@320
|
92
|
Chris@320
|
93 if (!m_plugin->initialise(channelCount,
|
Chris@350
|
94 m_transform.getStepSize(),
|
Chris@350
|
95 m_transform.getBlockSize())) {
|
Chris@361
|
96
|
Chris@361
|
97 size_t pstep = m_transform.getStepSize();
|
Chris@361
|
98 size_t pblock = m_transform.getBlockSize();
|
Chris@361
|
99
|
Chris@361
|
100 m_transform.setStepSize(0);
|
Chris@361
|
101 m_transform.setBlockSize(0);
|
Chris@361
|
102 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@361
|
103 (m_transform, m_plugin);
|
Chris@361
|
104
|
Chris@361
|
105 if (m_transform.getStepSize() != pstep ||
|
Chris@361
|
106 m_transform.getBlockSize() != pblock) {
|
Chris@361
|
107
|
Chris@361
|
108 if (!m_plugin->initialise(channelCount,
|
Chris@361
|
109 m_transform.getStepSize(),
|
Chris@361
|
110 m_transform.getBlockSize())) {
|
Chris@361
|
111
|
Chris@361
|
112 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
113 return;
|
Chris@361
|
114
|
Chris@361
|
115 } else {
|
Chris@361
|
116
|
Chris@361
|
117 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
|
Chris@361
|
118 .arg(pluginId)
|
Chris@361
|
119 .arg(pstep)
|
Chris@361
|
120 .arg(pblock)
|
Chris@361
|
121 .arg(m_transform.getStepSize())
|
Chris@361
|
122 .arg(m_transform.getBlockSize());
|
Chris@361
|
123 }
|
Chris@361
|
124
|
Chris@361
|
125 } else {
|
Chris@361
|
126
|
Chris@361
|
127 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
128 return;
|
Chris@361
|
129 }
|
Chris@320
|
130 }
|
Chris@320
|
131
|
Chris@366
|
132 if (m_transform.getPluginVersion() != "") {
|
Chris@366
|
133 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
|
Chris@366
|
134 if (pv != m_transform.getPluginVersion()) {
|
Chris@366
|
135 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
|
Chris@366
|
136 .arg(m_transform.getPluginVersion())
|
Chris@366
|
137 .arg(pluginId)
|
Chris@366
|
138 .arg(pv);
|
Chris@366
|
139 if (m_message != "") {
|
Chris@366
|
140 m_message = QString("%1; %2").arg(vm).arg(m_message);
|
Chris@366
|
141 } else {
|
Chris@366
|
142 m_message = vm;
|
Chris@366
|
143 }
|
Chris@366
|
144 }
|
Chris@366
|
145 }
|
Chris@366
|
146
|
Chris@320
|
147 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
|
Chris@320
|
148
|
Chris@320
|
149 if (outputs.empty()) {
|
Chris@361
|
150 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
|
Chris@320
|
151 return;
|
Chris@320
|
152 }
|
Chris@320
|
153
|
Chris@320
|
154 for (size_t i = 0; i < outputs.size(); ++i) {
|
Chris@687
|
155 // DEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl;
|
Chris@350
|
156 if (m_transform.getOutput() == "" ||
|
Chris@350
|
157 outputs[i].identifier == m_transform.getOutput().toStdString()) {
|
Chris@320
|
158 m_outputFeatureNo = i;
|
Chris@441
|
159 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
|
Chris@320
|
160 break;
|
Chris@320
|
161 }
|
Chris@320
|
162 }
|
Chris@320
|
163
|
Chris@320
|
164 if (!m_descriptor) {
|
Chris@361
|
165 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
|
Chris@361
|
166 .arg(pluginId)
|
Chris@361
|
167 .arg(m_transform.getOutput());
|
Chris@320
|
168 return;
|
Chris@320
|
169 }
|
Chris@320
|
170
|
Chris@558
|
171 createOutputModel();
|
Chris@558
|
172 }
|
Chris@558
|
173
|
Chris@558
|
174 void
|
Chris@558
|
175 FeatureExtractionModelTransformer::createOutputModel()
|
Chris@558
|
176 {
|
Chris@558
|
177 DenseTimeValueModel *input = getConformingInput();
|
Chris@558
|
178
|
Chris@331
|
179 // std::cerr << "FeatureExtractionModelTransformer: output sample type "
|
Chris@320
|
180 // << m_descriptor->sampleType << std::endl;
|
Chris@320
|
181
|
Chris@558
|
182 PluginRDFDescription description(m_transform.getPluginIdentifier());
|
Chris@558
|
183 QString outputId = m_transform.getOutput();
|
Chris@558
|
184
|
Chris@320
|
185 int binCount = 1;
|
Chris@320
|
186 float minValue = 0.0, maxValue = 0.0;
|
Chris@320
|
187 bool haveExtents = false;
|
Chris@320
|
188
|
Chris@320
|
189 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
190 binCount = m_descriptor->binCount;
|
Chris@320
|
191 }
|
Chris@320
|
192
|
Chris@331
|
193 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
|
Chris@320
|
194 // << binCount << std::endl;
|
Chris@320
|
195
|
Chris@320
|
196 if (binCount > 0 && m_descriptor->hasKnownExtents) {
|
Chris@320
|
197 minValue = m_descriptor->minValue;
|
Chris@320
|
198 maxValue = m_descriptor->maxValue;
|
Chris@320
|
199 haveExtents = true;
|
Chris@320
|
200 }
|
Chris@320
|
201
|
Chris@350
|
202 size_t modelRate = input->getSampleRate();
|
Chris@320
|
203 size_t modelResolution = 1;
|
Chris@320
|
204
|
Chris@320
|
205 switch (m_descriptor->sampleType) {
|
Chris@320
|
206
|
Chris@320
|
207 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
|
Chris@320
|
208 if (m_descriptor->sampleRate != 0.0) {
|
Chris@320
|
209 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
|
Chris@320
|
210 }
|
Chris@320
|
211 break;
|
Chris@320
|
212
|
Chris@320
|
213 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
|
Chris@350
|
214 modelResolution = m_transform.getStepSize();
|
Chris@320
|
215 break;
|
Chris@320
|
216
|
Chris@320
|
217 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
|
Chris@451
|
218 //!!! SV doesn't actually support display of models that have
|
Chris@451
|
219 //!!! different underlying rates together -- so we always set
|
Chris@451
|
220 //!!! the model rate to be the input model's rate, and adjust
|
Chris@451
|
221 //!!! the resolution appropriately. We can't properly display
|
Chris@451
|
222 //!!! data with a higher resolution than the base model at all
|
Chris@451
|
223 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
|
Chris@451
|
224 if (m_descriptor->sampleRate > input->getSampleRate()) {
|
Chris@451
|
225 modelResolution = 1;
|
Chris@451
|
226 } else {
|
Chris@451
|
227 modelResolution = size_t(input->getSampleRate() /
|
Chris@451
|
228 m_descriptor->sampleRate);
|
Chris@451
|
229 }
|
Chris@320
|
230 break;
|
Chris@320
|
231 }
|
Chris@320
|
232
|
Chris@441
|
233 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
|
Chris@441
|
234
|
Chris@441
|
235 if (binCount == 0 &&
|
Chris@441
|
236 (preDurationPlugin || !m_descriptor->hasDuration)) {
|
Chris@320
|
237
|
Chris@445
|
238 // Anything with no value and no duration is an instant
|
Chris@445
|
239
|
Chris@320
|
240 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
|
Chris@320
|
241 false);
|
Chris@320
|
242
|
Chris@558
|
243 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
|
Chris@558
|
244 m_output->setRDFTypeURI(outputEventTypeURI);
|
Chris@558
|
245
|
Chris@441
|
246 } else if ((preDurationPlugin && binCount > 1 &&
|
Chris@441
|
247 (m_descriptor->sampleType ==
|
Chris@441
|
248 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
|
Chris@441
|
249 (!preDurationPlugin && m_descriptor->hasDuration)) {
|
Chris@441
|
250
|
Chris@441
|
251 // For plugins using the old v1 API without explicit duration,
|
Chris@441
|
252 // we treat anything that has multiple bins (i.e. that has the
|
Chris@441
|
253 // potential to have value and duration) and a variable sample
|
Chris@441
|
254 // rate as a note model, taking its values as pitch, duration
|
Chris@441
|
255 // and velocity (if present) respectively. This is the same
|
Chris@441
|
256 // behaviour as always applied by SV to these plugins in the
|
Chris@441
|
257 // past.
|
Chris@441
|
258
|
Chris@441
|
259 // For plugins with the newer API, we treat anything with
|
Chris@441
|
260 // duration as either a note model with pitch and velocity, or
|
Chris@441
|
261 // a region model.
|
Chris@441
|
262
|
Chris@441
|
263 // How do we know whether it's an interval or note model?
|
Chris@441
|
264 // What's the essential difference? Is a note model any
|
Chris@441
|
265 // interval model using a Hz or "MIDI pitch" scale? There
|
Chris@441
|
266 // isn't really a reliable test for "MIDI pitch"... Does a
|
Chris@441
|
267 // note model always have velocity? This is a good question
|
Chris@441
|
268 // to be addressed by accompanying RDF, but for the moment we
|
Chris@441
|
269 // will do the following...
|
Chris@441
|
270
|
Chris@441
|
271 bool isNoteModel = false;
|
Chris@441
|
272
|
Chris@441
|
273 // Regions have only value (and duration -- we can't extract a
|
Chris@441
|
274 // region model from an old-style plugin that doesn't support
|
Chris@441
|
275 // duration)
|
Chris@441
|
276 if (binCount > 1) isNoteModel = true;
|
Chris@441
|
277
|
Chris@595
|
278 // Regions do not have units of Hz or MIDI things (a sweeping
|
Chris@595
|
279 // assumption!)
|
Chris@595
|
280 if (m_descriptor->unit == "Hz" ||
|
Chris@595
|
281 m_descriptor->unit.find("MIDI") != std::string::npos ||
|
Chris@595
|
282 m_descriptor->unit.find("midi") != std::string::npos) {
|
Chris@595
|
283 isNoteModel = true;
|
Chris@595
|
284 }
|
Chris@441
|
285
|
Chris@441
|
286 // If we had a "sparse 3D model", we would have the additional
|
Chris@441
|
287 // problem of determining whether to use that here (if bin
|
Chris@441
|
288 // count > 1). But we don't.
|
Chris@441
|
289
|
Chris@441
|
290 if (isNoteModel) {
|
Chris@441
|
291
|
Chris@441
|
292 NoteModel *model;
|
Chris@441
|
293 if (haveExtents) {
|
Chris@441
|
294 model = new NoteModel
|
Chris@441
|
295 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
296 } else {
|
Chris@441
|
297 model = new NoteModel
|
Chris@441
|
298 (modelRate, modelResolution, false);
|
Chris@441
|
299 }
|
Chris@441
|
300 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
301 m_output = model;
|
Chris@441
|
302
|
Chris@441
|
303 } else {
|
Chris@441
|
304
|
Chris@441
|
305 RegionModel *model;
|
Chris@441
|
306 if (haveExtents) {
|
Chris@441
|
307 model = new RegionModel
|
Chris@441
|
308 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
309 } else {
|
Chris@441
|
310 model = new RegionModel
|
Chris@441
|
311 (modelRate, modelResolution, false);
|
Chris@441
|
312 }
|
Chris@441
|
313 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
314 m_output = model;
|
Chris@441
|
315 }
|
Chris@441
|
316
|
Chris@558
|
317 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
|
Chris@558
|
318 m_output->setRDFTypeURI(outputEventTypeURI);
|
Chris@558
|
319
|
Chris@441
|
320 } else if (binCount == 1 ||
|
Chris@441
|
321 (m_descriptor->sampleType ==
|
Chris@441
|
322 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
|
Chris@441
|
323
|
Chris@441
|
324 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
325 // has only one value per result must be a sparse time value
|
Chris@441
|
326 // model.
|
Chris@441
|
327
|
Chris@441
|
328 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
329 // has a variable sample rate is also treated as a sparse time
|
Chris@441
|
330 // value model regardless of its bin count, because we lack a
|
Chris@441
|
331 // sparse 3D model.
|
Chris@320
|
332
|
Chris@320
|
333 SparseTimeValueModel *model;
|
Chris@320
|
334 if (haveExtents) {
|
Chris@320
|
335 model = new SparseTimeValueModel
|
Chris@320
|
336 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@320
|
337 } else {
|
Chris@320
|
338 model = new SparseTimeValueModel
|
Chris@320
|
339 (modelRate, modelResolution, false);
|
Chris@320
|
340 }
|
Chris@558
|
341
|
Chris@558
|
342 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
|
Chris@320
|
343 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
|
Chris@320
|
344
|
Chris@320
|
345 m_output = model;
|
Chris@320
|
346
|
Chris@558
|
347 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
|
Chris@558
|
348 m_output->setRDFTypeURI(outputEventTypeURI);
|
Chris@558
|
349
|
Chris@441
|
350 } else {
|
Chris@320
|
351
|
Chris@441
|
352 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
353 // has a fixed sample rate and more than one value per result
|
Chris@441
|
354 // must be a dense 3D model.
|
Chris@320
|
355
|
Chris@320
|
356 EditableDenseThreeDimensionalModel *model =
|
Chris@320
|
357 new EditableDenseThreeDimensionalModel
|
Chris@535
|
358 (modelRate, modelResolution, binCount,
|
Chris@535
|
359 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
|
Chris@535
|
360 false);
|
Chris@320
|
361
|
Chris@320
|
362 if (!m_descriptor->binNames.empty()) {
|
Chris@320
|
363 std::vector<QString> names;
|
Chris@320
|
364 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
|
Chris@320
|
365 names.push_back(m_descriptor->binNames[i].c_str());
|
Chris@320
|
366 }
|
Chris@320
|
367 model->setBinNames(names);
|
Chris@320
|
368 }
|
Chris@320
|
369
|
Chris@320
|
370 m_output = model;
|
Chris@558
|
371
|
Chris@558
|
372 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
|
Chris@558
|
373 m_output->setRDFTypeURI(outputSignalTypeURI);
|
Chris@320
|
374 }
|
Chris@333
|
375
|
Chris@350
|
376 if (m_output) m_output->setSourceModel(input);
|
Chris@320
|
377 }
|
Chris@320
|
378
|
Chris@331
|
379 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
|
Chris@320
|
380 {
|
Chris@687
|
381 // DEBUG << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << endl;
|
Chris@320
|
382 delete m_plugin;
|
Chris@320
|
383 delete m_descriptor;
|
Chris@320
|
384 }
|
Chris@320
|
385
|
Chris@320
|
386 DenseTimeValueModel *
|
Chris@350
|
387 FeatureExtractionModelTransformer::getConformingInput()
|
Chris@320
|
388 {
|
Chris@687
|
389 // DEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
|
Chris@408
|
390
|
Chris@320
|
391 DenseTimeValueModel *dtvm =
|
Chris@320
|
392 dynamic_cast<DenseTimeValueModel *>(getInputModel());
|
Chris@320
|
393 if (!dtvm) {
|
Chris@687
|
394 DEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
|
Chris@320
|
395 }
|
Chris@320
|
396 return dtvm;
|
Chris@320
|
397 }
|
Chris@320
|
398
|
Chris@320
|
399 void
|
Chris@331
|
400 FeatureExtractionModelTransformer::run()
|
Chris@320
|
401 {
|
Chris@350
|
402 DenseTimeValueModel *input = getConformingInput();
|
Chris@320
|
403 if (!input) return;
|
Chris@320
|
404
|
Chris@320
|
405 if (!m_output) return;
|
Chris@320
|
406
|
Chris@497
|
407 while (!input->isReady() && !m_abandoned) {
|
Chris@687
|
408 DEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
|
Chris@497
|
409 usleep(500000);
|
Chris@320
|
410 }
|
Chris@497
|
411 if (m_abandoned) return;
|
Chris@320
|
412
|
Chris@350
|
413 size_t sampleRate = input->getSampleRate();
|
Chris@320
|
414
|
Chris@320
|
415 size_t channelCount = input->getChannelCount();
|
Chris@320
|
416 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
417 channelCount = 1;
|
Chris@320
|
418 }
|
Chris@320
|
419
|
Chris@320
|
420 float **buffers = new float*[channelCount];
|
Chris@320
|
421 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
422 buffers[ch] = new float[m_transform.getBlockSize() + 2];
|
Chris@320
|
423 }
|
Chris@320
|
424
|
Chris@350
|
425 size_t stepSize = m_transform.getStepSize();
|
Chris@350
|
426 size_t blockSize = m_transform.getBlockSize();
|
Chris@350
|
427
|
Chris@320
|
428 bool frequencyDomain = (m_plugin->getInputDomain() ==
|
Chris@320
|
429 Vamp::Plugin::FrequencyDomain);
|
Chris@320
|
430 std::vector<FFTModel *> fftModels;
|
Chris@320
|
431
|
Chris@320
|
432 if (frequencyDomain) {
|
Chris@320
|
433 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
434 FFTModel *model = new FFTModel
|
Chris@350
|
435 (getConformingInput(),
|
Chris@350
|
436 channelCount == 1 ? m_input.getChannel() : ch,
|
Chris@350
|
437 m_transform.getWindowType(),
|
Chris@350
|
438 blockSize,
|
Chris@350
|
439 stepSize,
|
Chris@350
|
440 blockSize,
|
Chris@334
|
441 false,
|
Chris@334
|
442 StorageAdviser::PrecisionCritical);
|
Chris@320
|
443 if (!model->isOK()) {
|
Chris@320
|
444 delete model;
|
Chris@320
|
445 setCompletion(100);
|
Chris@387
|
446 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
|
Chris@387
|
447 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
|
Chris@320
|
448 }
|
Chris@320
|
449 model->resume();
|
Chris@320
|
450 fftModels.push_back(model);
|
Chris@320
|
451 }
|
Chris@320
|
452 }
|
Chris@320
|
453
|
Chris@350
|
454 long startFrame = m_input.getModel()->getStartFrame();
|
Chris@350
|
455 long endFrame = m_input.getModel()->getEndFrame();
|
Chris@320
|
456
|
Chris@350
|
457 RealTime contextStartRT = m_transform.getStartTime();
|
Chris@350
|
458 RealTime contextDurationRT = m_transform.getDuration();
|
Chris@350
|
459
|
Chris@350
|
460 long contextStart =
|
Chris@350
|
461 RealTime::realTime2Frame(contextStartRT, sampleRate);
|
Chris@350
|
462
|
Chris@350
|
463 long contextDuration =
|
Chris@350
|
464 RealTime::realTime2Frame(contextDurationRT, sampleRate);
|
Chris@320
|
465
|
Chris@320
|
466 if (contextStart == 0 || contextStart < startFrame) {
|
Chris@320
|
467 contextStart = startFrame;
|
Chris@320
|
468 }
|
Chris@320
|
469
|
Chris@320
|
470 if (contextDuration == 0) {
|
Chris@320
|
471 contextDuration = endFrame - contextStart;
|
Chris@320
|
472 }
|
Chris@320
|
473 if (contextStart + contextDuration > endFrame) {
|
Chris@320
|
474 contextDuration = endFrame - contextStart;
|
Chris@320
|
475 }
|
Chris@320
|
476
|
Chris@320
|
477 long blockFrame = contextStart;
|
Chris@320
|
478
|
Chris@320
|
479 long prevCompletion = 0;
|
Chris@320
|
480
|
Chris@320
|
481 setCompletion(0);
|
Chris@320
|
482
|
Chris@556
|
483 float *reals = 0;
|
Chris@556
|
484 float *imaginaries = 0;
|
Chris@556
|
485 if (frequencyDomain) {
|
Chris@556
|
486 reals = new float[blockSize/2 + 1];
|
Chris@556
|
487 imaginaries = new float[blockSize/2 + 1];
|
Chris@556
|
488 }
|
Chris@556
|
489
|
Chris@678
|
490 QString error = "";
|
Chris@678
|
491
|
Chris@320
|
492 while (!m_abandoned) {
|
Chris@320
|
493
|
Chris@320
|
494 if (frequencyDomain) {
|
Chris@350
|
495 if (blockFrame - int(blockSize)/2 >
|
Chris@320
|
496 contextStart + contextDuration) break;
|
Chris@320
|
497 } else {
|
Chris@320
|
498 if (blockFrame >=
|
Chris@320
|
499 contextStart + contextDuration) break;
|
Chris@320
|
500 }
|
Chris@320
|
501
|
Chris@687
|
502 // DEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
|
Chris@320
|
503 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
|
Chris@687
|
504 // << blockSize << endl;
|
Chris@320
|
505
|
Chris@320
|
506 long completion =
|
Chris@350
|
507 (((blockFrame - contextStart) / stepSize) * 99) /
|
Chris@557
|
508 (contextDuration / stepSize + 1);
|
Chris@320
|
509
|
Chris@350
|
510 // channelCount is either m_input.getModel()->channelCount or 1
|
Chris@320
|
511
|
Chris@363
|
512 if (frequencyDomain) {
|
Chris@363
|
513 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
514 int column = (blockFrame - startFrame) / stepSize;
|
Chris@556
|
515 fftModels[ch]->getValuesAt(column, reals, imaginaries);
|
Chris@350
|
516 for (size_t i = 0; i <= blockSize/2; ++i) {
|
Chris@556
|
517 buffers[ch][i*2] = reals[i];
|
Chris@556
|
518 buffers[ch][i*2+1] = imaginaries[i];
|
Chris@320
|
519 }
|
Chris@678
|
520 error = fftModels[ch]->getError();
|
Chris@678
|
521 if (error != "") {
|
Chris@686
|
522 std::cerr << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << std::endl;
|
Chris@678
|
523 m_abandoned = true;
|
Chris@678
|
524 m_message = error;
|
Chris@678
|
525 }
|
Chris@363
|
526 }
|
Chris@363
|
527 } else {
|
Chris@363
|
528 getFrames(channelCount, blockFrame, blockSize, buffers);
|
Chris@320
|
529 }
|
Chris@320
|
530
|
Chris@497
|
531 if (m_abandoned) break;
|
Chris@497
|
532
|
Chris@320
|
533 Vamp::Plugin::FeatureSet features = m_plugin->process
|
Chris@320
|
534 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
|
Chris@320
|
535
|
Chris@497
|
536 if (m_abandoned) break;
|
Chris@497
|
537
|
Chris@320
|
538 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@320
|
539 Vamp::Plugin::Feature feature =
|
Chris@320
|
540 features[m_outputFeatureNo][fi];
|
Chris@320
|
541 addFeature(blockFrame, feature);
|
Chris@320
|
542 }
|
Chris@320
|
543
|
Chris@320
|
544 if (blockFrame == contextStart || completion > prevCompletion) {
|
Chris@320
|
545 setCompletion(completion);
|
Chris@320
|
546 prevCompletion = completion;
|
Chris@320
|
547 }
|
Chris@320
|
548
|
Chris@350
|
549 blockFrame += stepSize;
|
Chris@320
|
550 }
|
Chris@320
|
551
|
Chris@497
|
552 if (!m_abandoned) {
|
Chris@497
|
553 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
|
Chris@320
|
554
|
Chris@497
|
555 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@497
|
556 Vamp::Plugin::Feature feature =
|
Chris@497
|
557 features[m_outputFeatureNo][fi];
|
Chris@497
|
558 addFeature(blockFrame, feature);
|
Chris@497
|
559 }
|
Chris@497
|
560 }
|
Chris@320
|
561
|
Chris@497
|
562 setCompletion(100);
|
Chris@320
|
563
|
Chris@320
|
564 if (frequencyDomain) {
|
Chris@320
|
565 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
566 delete fftModels[ch];
|
Chris@320
|
567 }
|
Chris@556
|
568 delete[] reals;
|
Chris@556
|
569 delete[] imaginaries;
|
Chris@320
|
570 }
|
Chris@320
|
571 }
|
Chris@320
|
572
|
Chris@320
|
573 void
|
Chris@363
|
574 FeatureExtractionModelTransformer::getFrames(int channelCount,
|
Chris@363
|
575 long startFrame, long size,
|
Chris@363
|
576 float **buffers)
|
Chris@320
|
577 {
|
Chris@320
|
578 long offset = 0;
|
Chris@320
|
579
|
Chris@320
|
580 if (startFrame < 0) {
|
Chris@363
|
581 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
582 for (int i = 0; i < size && startFrame + i < 0; ++i) {
|
Chris@363
|
583 buffers[c][i] = 0.0f;
|
Chris@363
|
584 }
|
Chris@320
|
585 }
|
Chris@320
|
586 offset = -startFrame;
|
Chris@320
|
587 size -= offset;
|
Chris@320
|
588 if (size <= 0) return;
|
Chris@320
|
589 startFrame = 0;
|
Chris@320
|
590 }
|
Chris@320
|
591
|
Chris@350
|
592 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
593 if (!input) return;
|
Chris@363
|
594
|
Chris@363
|
595 long got = 0;
|
Chris@350
|
596
|
Chris@363
|
597 if (channelCount == 1) {
|
Chris@363
|
598
|
Chris@363
|
599 got = input->getData(m_input.getChannel(), startFrame, size,
|
Chris@363
|
600 buffers[0] + offset);
|
Chris@363
|
601
|
Chris@363
|
602 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
|
Chris@363
|
603 // use mean instead of sum, as plugin input
|
Chris@363
|
604 float cc = float(input->getChannelCount());
|
Chris@363
|
605 for (long i = 0; i < size; ++i) {
|
Chris@363
|
606 buffers[0][i + offset] /= cc;
|
Chris@363
|
607 }
|
Chris@363
|
608 }
|
Chris@363
|
609
|
Chris@363
|
610 } else {
|
Chris@363
|
611
|
Chris@363
|
612 float **writebuf = buffers;
|
Chris@363
|
613 if (offset > 0) {
|
Chris@363
|
614 writebuf = new float *[channelCount];
|
Chris@363
|
615 for (int i = 0; i < channelCount; ++i) {
|
Chris@363
|
616 writebuf[i] = buffers[i] + offset;
|
Chris@363
|
617 }
|
Chris@363
|
618 }
|
Chris@363
|
619
|
Chris@363
|
620 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
|
Chris@363
|
621
|
Chris@363
|
622 if (writebuf != buffers) delete[] writebuf;
|
Chris@363
|
623 }
|
Chris@320
|
624
|
Chris@320
|
625 while (got < size) {
|
Chris@363
|
626 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
627 buffers[c][got + offset] = 0.0;
|
Chris@363
|
628 }
|
Chris@320
|
629 ++got;
|
Chris@320
|
630 }
|
Chris@320
|
631 }
|
Chris@320
|
632
|
Chris@320
|
633 void
|
Chris@331
|
634 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
|
Chris@320
|
635 const Vamp::Plugin::Feature &feature)
|
Chris@320
|
636 {
|
Chris@350
|
637 size_t inputRate = m_input.getModel()->getSampleRate();
|
Chris@320
|
638
|
Chris@687
|
639 // DEBUG << "FeatureExtractionModelTransformer::addFeature("
|
Chris@687
|
640 // << blockFrame << ")" << endl;
|
Chris@320
|
641
|
Chris@320
|
642 int binCount = 1;
|
Chris@320
|
643 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
644 binCount = m_descriptor->binCount;
|
Chris@320
|
645 }
|
Chris@320
|
646
|
Chris@320
|
647 size_t frame = blockFrame;
|
Chris@320
|
648
|
Chris@320
|
649 if (m_descriptor->sampleType ==
|
Chris@320
|
650 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@320
|
651
|
Chris@320
|
652 if (!feature.hasTimestamp) {
|
Chris@320
|
653 std::cerr
|
Chris@331
|
654 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
|
Chris@320
|
655 << "Feature has variable sample rate but no timestamp!"
|
Chris@320
|
656 << std::endl;
|
Chris@320
|
657 return;
|
Chris@320
|
658 } else {
|
Chris@320
|
659 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
|
Chris@320
|
660 }
|
Chris@320
|
661
|
Chris@320
|
662 } else if (m_descriptor->sampleType ==
|
Chris@320
|
663 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
|
Chris@320
|
664
|
Chris@320
|
665 if (feature.hasTimestamp) {
|
Chris@320
|
666 //!!! warning: sampleRate may be non-integral
|
Chris@320
|
667 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
|
Chris@451
|
668 //!!! see comment above when setting up modelResolution and modelRate
|
Chris@451
|
669 // lrintf(m_descriptor->sampleRate));
|
Chris@451
|
670 inputRate);
|
Chris@320
|
671 } else {
|
Chris@320
|
672 frame = m_output->getEndFrame();
|
Chris@320
|
673 }
|
Chris@320
|
674 }
|
Chris@320
|
675
|
Chris@441
|
676 // Rather than repeat the complicated tests from the constructor
|
Chris@441
|
677 // to determine what sort of model we must be adding the features
|
Chris@441
|
678 // to, we instead test what sort of model the constructor decided
|
Chris@441
|
679 // to create.
|
Chris@320
|
680
|
Chris@441
|
681 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@441
|
682
|
Chris@441
|
683 SparseOneDimensionalModel *model =
|
Chris@350
|
684 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
685 if (!model) return;
|
Chris@350
|
686
|
Chris@441
|
687 model->addPoint(SparseOneDimensionalModel::Point
|
Chris@441
|
688 (frame, feature.label.c_str()));
|
Chris@320
|
689
|
Chris@441
|
690 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
691
|
Chris@350
|
692 SparseTimeValueModel *model =
|
Chris@350
|
693 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
694 if (!model) return;
|
Chris@350
|
695
|
Chris@454
|
696 for (int i = 0; i < feature.values.size(); ++i) {
|
Chris@454
|
697
|
Chris@454
|
698 float value = feature.values[i];
|
Chris@454
|
699
|
Chris@454
|
700 QString label = feature.label.c_str();
|
Chris@454
|
701 if (feature.values.size() > 1) {
|
Chris@454
|
702 label = QString("[%1] %2").arg(i+1).arg(label);
|
Chris@454
|
703 }
|
Chris@454
|
704
|
Chris@454
|
705 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
|
Chris@454
|
706 }
|
Chris@320
|
707
|
Chris@441
|
708 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
|
Chris@320
|
709
|
Chris@441
|
710 int index = 0;
|
Chris@441
|
711
|
Chris@441
|
712 float value = 0.0;
|
Chris@441
|
713 if (feature.values.size() > index) {
|
Chris@441
|
714 value = feature.values[index++];
|
Chris@441
|
715 }
|
Chris@320
|
716
|
Chris@320
|
717 float duration = 1;
|
Chris@441
|
718 if (feature.hasDuration) {
|
Chris@441
|
719 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
|
Chris@441
|
720 } else {
|
Chris@441
|
721 if (feature.values.size() > index) {
|
Chris@441
|
722 duration = feature.values[index++];
|
Chris@441
|
723 }
|
Chris@441
|
724 }
|
Chris@320
|
725
|
Chris@441
|
726 if (isOutput<NoteModel>()) {
|
Chris@320
|
727
|
Chris@441
|
728 float velocity = 100;
|
Chris@441
|
729 if (feature.values.size() > index) {
|
Chris@441
|
730 velocity = feature.values[index++];
|
Chris@441
|
731 }
|
Chris@441
|
732 if (velocity < 0) velocity = 127;
|
Chris@441
|
733 if (velocity > 127) velocity = 127;
|
Chris@320
|
734
|
Chris@441
|
735 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@441
|
736 if (!model) return;
|
Chris@441
|
737 model->addPoint(NoteModel::Point(frame, value, // value is pitch
|
Chris@441
|
738 lrintf(duration),
|
Chris@441
|
739 velocity / 127.f,
|
Chris@441
|
740 feature.label.c_str()));
|
Chris@441
|
741 } else {
|
Chris@441
|
742 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@454
|
743 if (!model) return;
|
Chris@454
|
744
|
Chris@474
|
745 if (feature.hasDuration && !feature.values.empty()) {
|
Chris@454
|
746
|
Chris@454
|
747 for (int i = 0; i < feature.values.size(); ++i) {
|
Chris@454
|
748
|
Chris@454
|
749 float value = feature.values[i];
|
Chris@454
|
750
|
Chris@454
|
751 QString label = feature.label.c_str();
|
Chris@454
|
752 if (feature.values.size() > 1) {
|
Chris@454
|
753 label = QString("[%1] %2").arg(i+1).arg(label);
|
Chris@454
|
754 }
|
Chris@454
|
755
|
Chris@454
|
756 model->addPoint(RegionModel::Point(frame, value,
|
Chris@454
|
757 lrintf(duration),
|
Chris@454
|
758 label));
|
Chris@454
|
759 }
|
Chris@454
|
760 } else {
|
Chris@454
|
761
|
Chris@441
|
762 model->addPoint(RegionModel::Point(frame, value,
|
Chris@441
|
763 lrintf(duration),
|
Chris@441
|
764 feature.label.c_str()));
|
Chris@454
|
765 }
|
Chris@441
|
766 }
|
Chris@320
|
767
|
Chris@441
|
768 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
769
|
Chris@533
|
770 DenseThreeDimensionalModel::Column values =
|
Chris@533
|
771 DenseThreeDimensionalModel::Column::fromStdVector(feature.values);
|
Chris@320
|
772
|
Chris@320
|
773 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
774 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
775 if (!model) return;
|
Chris@320
|
776
|
Chris@320
|
777 model->setColumn(frame / model->getResolution(), values);
|
Chris@441
|
778
|
Chris@441
|
779 } else {
|
Chris@687
|
780 DEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
|
Chris@320
|
781 }
|
Chris@320
|
782 }
|
Chris@320
|
783
|
Chris@320
|
784 void
|
Chris@331
|
785 FeatureExtractionModelTransformer::setCompletion(int completion)
|
Chris@320
|
786 {
|
Chris@320
|
787 int binCount = 1;
|
Chris@320
|
788 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
789 binCount = m_descriptor->binCount;
|
Chris@320
|
790 }
|
Chris@320
|
791
|
Chris@687
|
792 // DEBUG << "FeatureExtractionModelTransformer::setCompletion("
|
Chris@687
|
793 // << completion << ")" << endl;
|
Chris@320
|
794
|
Chris@441
|
795 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@320
|
796
|
Chris@350
|
797 SparseOneDimensionalModel *model =
|
Chris@350
|
798 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
799 if (!model) return;
|
Chris@441
|
800 model->setCompletion(completion, true);
|
Chris@320
|
801
|
Chris@441
|
802 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
803
|
Chris@350
|
804 SparseTimeValueModel *model =
|
Chris@350
|
805 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
806 if (!model) return;
|
Chris@441
|
807 model->setCompletion(completion, true);
|
Chris@320
|
808
|
Chris@441
|
809 } else if (isOutput<NoteModel>()) {
|
Chris@320
|
810
|
Chris@441
|
811 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@320
|
812 if (!model) return;
|
Chris@441
|
813 model->setCompletion(completion, true);
|
Chris@320
|
814
|
Chris@441
|
815 } else if (isOutput<RegionModel>()) {
|
Chris@441
|
816
|
Chris@441
|
817 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@441
|
818 if (!model) return;
|
Chris@441
|
819 model->setCompletion(completion, true);
|
Chris@441
|
820
|
Chris@441
|
821 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
822
|
Chris@320
|
823 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
824 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
825 if (!model) return;
|
Chris@350
|
826 model->setCompletion(completion, true); //!!!m_context.updates);
|
Chris@320
|
827 }
|
Chris@320
|
828 }
|
Chris@320
|
829
|