Chris@320
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@320
|
2
|
Chris@320
|
3 /*
|
Chris@320
|
4 Sonic Visualiser
|
Chris@320
|
5 An audio file viewer and annotation editor.
|
Chris@320
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@320
|
7 This file copyright 2006 Chris Cannam and QMUL.
|
Chris@320
|
8
|
Chris@320
|
9 This program is free software; you can redistribute it and/or
|
Chris@320
|
10 modify it under the terms of the GNU General Public License as
|
Chris@320
|
11 published by the Free Software Foundation; either version 2 of the
|
Chris@320
|
12 License, or (at your option) any later version. See the file
|
Chris@320
|
13 COPYING included with this distribution for more information.
|
Chris@320
|
14 */
|
Chris@320
|
15
|
Chris@331
|
16 #include "FeatureExtractionModelTransformer.h"
|
Chris@320
|
17
|
Chris@320
|
18 #include "plugin/FeatureExtractionPluginFactory.h"
|
Chris@320
|
19 #include "plugin/PluginXml.h"
|
Chris@475
|
20 #include <vamp-hostsdk/Plugin.h>
|
Chris@320
|
21
|
Chris@320
|
22 #include "data/model/Model.h"
|
Chris@320
|
23 #include "base/Window.h"
|
Chris@387
|
24 #include "base/Exceptions.h"
|
Chris@320
|
25 #include "data/model/SparseOneDimensionalModel.h"
|
Chris@320
|
26 #include "data/model/SparseTimeValueModel.h"
|
Chris@320
|
27 #include "data/model/EditableDenseThreeDimensionalModel.h"
|
Chris@320
|
28 #include "data/model/DenseTimeValueModel.h"
|
Chris@320
|
29 #include "data/model/NoteModel.h"
|
Chris@441
|
30 #include "data/model/RegionModel.h"
|
Chris@320
|
31 #include "data/model/FFTModel.h"
|
Chris@320
|
32 #include "data/model/WaveFileModel.h"
|
Chris@558
|
33 #include "rdf/PluginRDFDescription.h"
|
Chris@320
|
34
|
Chris@350
|
35 #include "TransformFactory.h"
|
Chris@350
|
36
|
Chris@320
|
37 #include <iostream>
|
Chris@320
|
38
|
Chris@350
|
39 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
|
Chris@350
|
40 const Transform &transform) :
|
Chris@350
|
41 ModelTransformer(in, transform),
|
Chris@320
|
42 m_plugin(0),
|
Chris@320
|
43 m_descriptor(0),
|
Chris@778
|
44 m_outputNo(0)
|
Chris@320
|
45 {
|
Chris@690
|
46 // SVDEBUG << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId << ", outputName " << m_transform.getOutput() << endl;
|
Chris@350
|
47
|
Chris@350
|
48 QString pluginId = transform.getPluginIdentifier();
|
Chris@320
|
49
|
Chris@320
|
50 FeatureExtractionPluginFactory *factory =
|
Chris@320
|
51 FeatureExtractionPluginFactory::instanceFor(pluginId);
|
Chris@320
|
52
|
Chris@320
|
53 if (!factory) {
|
Chris@361
|
54 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
|
Chris@320
|
55 return;
|
Chris@320
|
56 }
|
Chris@320
|
57
|
Chris@350
|
58 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
59 if (!input) {
|
Chris@361
|
60 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
|
Chris@350
|
61 return;
|
Chris@350
|
62 }
|
Chris@320
|
63
|
Chris@350
|
64 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
|
Chris@320
|
65 if (!m_plugin) {
|
Chris@361
|
66 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
|
Chris@320
|
67 return;
|
Chris@320
|
68 }
|
Chris@320
|
69
|
Chris@350
|
70 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@350
|
71 (m_transform, m_plugin);
|
Chris@343
|
72
|
Chris@350
|
73 TransformFactory::getInstance()->setPluginParameters
|
Chris@350
|
74 (m_transform, m_plugin);
|
Chris@320
|
75
|
Chris@320
|
76 size_t channelCount = input->getChannelCount();
|
Chris@320
|
77 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
78 channelCount = 1;
|
Chris@320
|
79 }
|
Chris@320
|
80 if (m_plugin->getMinChannelCount() > channelCount) {
|
Chris@361
|
81 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
|
Chris@361
|
82 .arg(pluginId)
|
Chris@361
|
83 .arg(m_plugin->getMinChannelCount())
|
Chris@361
|
84 .arg(m_plugin->getMaxChannelCount())
|
Chris@361
|
85 .arg(input->getChannelCount());
|
Chris@320
|
86 return;
|
Chris@320
|
87 }
|
Chris@320
|
88
|
Chris@690
|
89 SVDEBUG << "Initialising feature extraction plugin with channels = "
|
Chris@350
|
90 << channelCount << ", step = " << m_transform.getStepSize()
|
Chris@687
|
91 << ", block = " << m_transform.getBlockSize() << endl;
|
Chris@320
|
92
|
Chris@320
|
93 if (!m_plugin->initialise(channelCount,
|
Chris@350
|
94 m_transform.getStepSize(),
|
Chris@350
|
95 m_transform.getBlockSize())) {
|
Chris@361
|
96
|
Chris@361
|
97 size_t pstep = m_transform.getStepSize();
|
Chris@361
|
98 size_t pblock = m_transform.getBlockSize();
|
Chris@361
|
99
|
Chris@361
|
100 m_transform.setStepSize(0);
|
Chris@361
|
101 m_transform.setBlockSize(0);
|
Chris@361
|
102 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@361
|
103 (m_transform, m_plugin);
|
Chris@361
|
104
|
Chris@361
|
105 if (m_transform.getStepSize() != pstep ||
|
Chris@361
|
106 m_transform.getBlockSize() != pblock) {
|
Chris@361
|
107
|
Chris@361
|
108 if (!m_plugin->initialise(channelCount,
|
Chris@361
|
109 m_transform.getStepSize(),
|
Chris@361
|
110 m_transform.getBlockSize())) {
|
Chris@361
|
111
|
Chris@361
|
112 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
113 return;
|
Chris@361
|
114
|
Chris@361
|
115 } else {
|
Chris@361
|
116
|
Chris@361
|
117 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
|
Chris@361
|
118 .arg(pluginId)
|
Chris@361
|
119 .arg(pstep)
|
Chris@361
|
120 .arg(pblock)
|
Chris@361
|
121 .arg(m_transform.getStepSize())
|
Chris@361
|
122 .arg(m_transform.getBlockSize());
|
Chris@361
|
123 }
|
Chris@361
|
124
|
Chris@361
|
125 } else {
|
Chris@361
|
126
|
Chris@361
|
127 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
128 return;
|
Chris@361
|
129 }
|
Chris@320
|
130 }
|
Chris@320
|
131
|
Chris@366
|
132 if (m_transform.getPluginVersion() != "") {
|
Chris@366
|
133 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
|
Chris@366
|
134 if (pv != m_transform.getPluginVersion()) {
|
Chris@366
|
135 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
|
Chris@366
|
136 .arg(m_transform.getPluginVersion())
|
Chris@366
|
137 .arg(pluginId)
|
Chris@366
|
138 .arg(pv);
|
Chris@366
|
139 if (m_message != "") {
|
Chris@366
|
140 m_message = QString("%1; %2").arg(vm).arg(m_message);
|
Chris@366
|
141 } else {
|
Chris@366
|
142 m_message = vm;
|
Chris@366
|
143 }
|
Chris@366
|
144 }
|
Chris@366
|
145 }
|
Chris@366
|
146
|
Chris@320
|
147 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
|
Chris@320
|
148
|
Chris@320
|
149 if (outputs.empty()) {
|
Chris@361
|
150 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
|
Chris@320
|
151 return;
|
Chris@320
|
152 }
|
Chris@320
|
153
|
Chris@320
|
154 for (size_t i = 0; i < outputs.size(); ++i) {
|
Chris@690
|
155 // SVDEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl;
|
Chris@350
|
156 if (m_transform.getOutput() == "" ||
|
Chris@350
|
157 outputs[i].identifier == m_transform.getOutput().toStdString()) {
|
Chris@778
|
158 m_outputNo = i;
|
Chris@441
|
159 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
|
Chris@320
|
160 break;
|
Chris@320
|
161 }
|
Chris@320
|
162 }
|
Chris@320
|
163
|
Chris@320
|
164 if (!m_descriptor) {
|
Chris@361
|
165 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
|
Chris@361
|
166 .arg(pluginId)
|
Chris@361
|
167 .arg(m_transform.getOutput());
|
Chris@320
|
168 return;
|
Chris@320
|
169 }
|
Chris@320
|
170
|
Chris@558
|
171 createOutputModel();
|
Chris@558
|
172 }
|
Chris@558
|
173
|
Chris@558
|
174 void
|
Chris@558
|
175 FeatureExtractionModelTransformer::createOutputModel()
|
Chris@558
|
176 {
|
Chris@558
|
177 DenseTimeValueModel *input = getConformingInput();
|
Chris@558
|
178
|
Chris@712
|
179 // std::cerr << "FeatureExtractionModelTransformer::createOutputModel: sample type " << m_descriptor->sampleType << ", rate " << m_descriptor->sampleRate << std::endl;
|
Chris@712
|
180
|
Chris@558
|
181 PluginRDFDescription description(m_transform.getPluginIdentifier());
|
Chris@558
|
182 QString outputId = m_transform.getOutput();
|
Chris@558
|
183
|
Chris@320
|
184 int binCount = 1;
|
Chris@320
|
185 float minValue = 0.0, maxValue = 0.0;
|
Chris@320
|
186 bool haveExtents = false;
|
Chris@320
|
187
|
Chris@320
|
188 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
189 binCount = m_descriptor->binCount;
|
Chris@320
|
190 }
|
Chris@320
|
191
|
Chris@331
|
192 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
|
Chris@320
|
193 // << binCount << std::endl;
|
Chris@320
|
194
|
Chris@320
|
195 if (binCount > 0 && m_descriptor->hasKnownExtents) {
|
Chris@320
|
196 minValue = m_descriptor->minValue;
|
Chris@320
|
197 maxValue = m_descriptor->maxValue;
|
Chris@320
|
198 haveExtents = true;
|
Chris@320
|
199 }
|
Chris@320
|
200
|
Chris@350
|
201 size_t modelRate = input->getSampleRate();
|
Chris@320
|
202 size_t modelResolution = 1;
|
Chris@712
|
203
|
Chris@320
|
204 switch (m_descriptor->sampleType) {
|
Chris@320
|
205
|
Chris@320
|
206 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
|
Chris@320
|
207 if (m_descriptor->sampleRate != 0.0) {
|
Chris@320
|
208 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
|
Chris@320
|
209 }
|
Chris@320
|
210 break;
|
Chris@320
|
211
|
Chris@320
|
212 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
|
Chris@350
|
213 modelResolution = m_transform.getStepSize();
|
Chris@320
|
214 break;
|
Chris@320
|
215
|
Chris@320
|
216 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
|
Chris@451
|
217 //!!! SV doesn't actually support display of models that have
|
Chris@451
|
218 //!!! different underlying rates together -- so we always set
|
Chris@451
|
219 //!!! the model rate to be the input model's rate, and adjust
|
Chris@451
|
220 //!!! the resolution appropriately. We can't properly display
|
Chris@451
|
221 //!!! data with a higher resolution than the base model at all
|
Chris@451
|
222 // modelRate = size_t(m_descriptor->sampleRate + 0.001);
|
Chris@451
|
223 if (m_descriptor->sampleRate > input->getSampleRate()) {
|
Chris@451
|
224 modelResolution = 1;
|
Chris@451
|
225 } else {
|
Chris@451
|
226 modelResolution = size_t(input->getSampleRate() /
|
Chris@451
|
227 m_descriptor->sampleRate);
|
Chris@451
|
228 }
|
Chris@320
|
229 break;
|
Chris@320
|
230 }
|
Chris@320
|
231
|
Chris@441
|
232 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
|
Chris@441
|
233
|
Chris@441
|
234 if (binCount == 0 &&
|
Chris@441
|
235 (preDurationPlugin || !m_descriptor->hasDuration)) {
|
Chris@320
|
236
|
Chris@445
|
237 // Anything with no value and no duration is an instant
|
Chris@445
|
238
|
Chris@320
|
239 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
|
Chris@320
|
240 false);
|
Chris@320
|
241
|
Chris@558
|
242 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
|
Chris@558
|
243 m_output->setRDFTypeURI(outputEventTypeURI);
|
Chris@558
|
244
|
Chris@441
|
245 } else if ((preDurationPlugin && binCount > 1 &&
|
Chris@441
|
246 (m_descriptor->sampleType ==
|
Chris@441
|
247 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
|
Chris@441
|
248 (!preDurationPlugin && m_descriptor->hasDuration)) {
|
Chris@441
|
249
|
Chris@441
|
250 // For plugins using the old v1 API without explicit duration,
|
Chris@441
|
251 // we treat anything that has multiple bins (i.e. that has the
|
Chris@441
|
252 // potential to have value and duration) and a variable sample
|
Chris@441
|
253 // rate as a note model, taking its values as pitch, duration
|
Chris@441
|
254 // and velocity (if present) respectively. This is the same
|
Chris@441
|
255 // behaviour as always applied by SV to these plugins in the
|
Chris@441
|
256 // past.
|
Chris@441
|
257
|
Chris@441
|
258 // For plugins with the newer API, we treat anything with
|
Chris@441
|
259 // duration as either a note model with pitch and velocity, or
|
Chris@441
|
260 // a region model.
|
Chris@441
|
261
|
Chris@441
|
262 // How do we know whether it's an interval or note model?
|
Chris@441
|
263 // What's the essential difference? Is a note model any
|
Chris@441
|
264 // interval model using a Hz or "MIDI pitch" scale? There
|
Chris@441
|
265 // isn't really a reliable test for "MIDI pitch"... Does a
|
Chris@441
|
266 // note model always have velocity? This is a good question
|
Chris@441
|
267 // to be addressed by accompanying RDF, but for the moment we
|
Chris@441
|
268 // will do the following...
|
Chris@441
|
269
|
Chris@441
|
270 bool isNoteModel = false;
|
Chris@441
|
271
|
Chris@441
|
272 // Regions have only value (and duration -- we can't extract a
|
Chris@441
|
273 // region model from an old-style plugin that doesn't support
|
Chris@441
|
274 // duration)
|
Chris@441
|
275 if (binCount > 1) isNoteModel = true;
|
Chris@441
|
276
|
Chris@595
|
277 // Regions do not have units of Hz or MIDI things (a sweeping
|
Chris@595
|
278 // assumption!)
|
Chris@595
|
279 if (m_descriptor->unit == "Hz" ||
|
Chris@595
|
280 m_descriptor->unit.find("MIDI") != std::string::npos ||
|
Chris@595
|
281 m_descriptor->unit.find("midi") != std::string::npos) {
|
Chris@595
|
282 isNoteModel = true;
|
Chris@595
|
283 }
|
Chris@441
|
284
|
Chris@441
|
285 // If we had a "sparse 3D model", we would have the additional
|
Chris@441
|
286 // problem of determining whether to use that here (if bin
|
Chris@441
|
287 // count > 1). But we don't.
|
Chris@441
|
288
|
Chris@441
|
289 if (isNoteModel) {
|
Chris@441
|
290
|
Chris@441
|
291 NoteModel *model;
|
Chris@441
|
292 if (haveExtents) {
|
Chris@441
|
293 model = new NoteModel
|
Chris@441
|
294 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
295 } else {
|
Chris@441
|
296 model = new NoteModel
|
Chris@441
|
297 (modelRate, modelResolution, false);
|
Chris@441
|
298 }
|
Chris@441
|
299 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
300 m_output = model;
|
Chris@441
|
301
|
Chris@441
|
302 } else {
|
Chris@441
|
303
|
Chris@441
|
304 RegionModel *model;
|
Chris@441
|
305 if (haveExtents) {
|
Chris@441
|
306 model = new RegionModel
|
Chris@441
|
307 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
308 } else {
|
Chris@441
|
309 model = new RegionModel
|
Chris@441
|
310 (modelRate, modelResolution, false);
|
Chris@441
|
311 }
|
Chris@441
|
312 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
313 m_output = model;
|
Chris@441
|
314 }
|
Chris@441
|
315
|
Chris@558
|
316 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
|
Chris@558
|
317 m_output->setRDFTypeURI(outputEventTypeURI);
|
Chris@558
|
318
|
Chris@441
|
319 } else if (binCount == 1 ||
|
Chris@441
|
320 (m_descriptor->sampleType ==
|
Chris@441
|
321 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
|
Chris@441
|
322
|
Chris@441
|
323 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
324 // has only one value per result must be a sparse time value
|
Chris@441
|
325 // model.
|
Chris@441
|
326
|
Chris@441
|
327 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
328 // has a variable sample rate is also treated as a sparse time
|
Chris@441
|
329 // value model regardless of its bin count, because we lack a
|
Chris@441
|
330 // sparse 3D model.
|
Chris@320
|
331
|
Chris@320
|
332 SparseTimeValueModel *model;
|
Chris@320
|
333 if (haveExtents) {
|
Chris@320
|
334 model = new SparseTimeValueModel
|
Chris@320
|
335 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@320
|
336 } else {
|
Chris@320
|
337 model = new SparseTimeValueModel
|
Chris@320
|
338 (modelRate, modelResolution, false);
|
Chris@320
|
339 }
|
Chris@558
|
340
|
Chris@558
|
341 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
|
Chris@778
|
342 model->setScaleUnits(outputs[m_outputNo].unit.c_str());
|
Chris@320
|
343
|
Chris@320
|
344 m_output = model;
|
Chris@320
|
345
|
Chris@558
|
346 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
|
Chris@558
|
347 m_output->setRDFTypeURI(outputEventTypeURI);
|
Chris@558
|
348
|
Chris@441
|
349 } else {
|
Chris@320
|
350
|
Chris@441
|
351 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
352 // has a fixed sample rate and more than one value per result
|
Chris@441
|
353 // must be a dense 3D model.
|
Chris@320
|
354
|
Chris@320
|
355 EditableDenseThreeDimensionalModel *model =
|
Chris@320
|
356 new EditableDenseThreeDimensionalModel
|
Chris@535
|
357 (modelRate, modelResolution, binCount,
|
Chris@535
|
358 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
|
Chris@535
|
359 false);
|
Chris@320
|
360
|
Chris@320
|
361 if (!m_descriptor->binNames.empty()) {
|
Chris@320
|
362 std::vector<QString> names;
|
Chris@320
|
363 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
|
Chris@320
|
364 names.push_back(m_descriptor->binNames[i].c_str());
|
Chris@320
|
365 }
|
Chris@320
|
366 model->setBinNames(names);
|
Chris@320
|
367 }
|
Chris@320
|
368
|
Chris@320
|
369 m_output = model;
|
Chris@558
|
370
|
Chris@558
|
371 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
|
Chris@558
|
372 m_output->setRDFTypeURI(outputSignalTypeURI);
|
Chris@320
|
373 }
|
Chris@333
|
374
|
Chris@350
|
375 if (m_output) m_output->setSourceModel(input);
|
Chris@320
|
376 }
|
Chris@320
|
377
|
Chris@331
|
378 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
|
Chris@320
|
379 {
|
Chris@690
|
380 // SVDEBUG << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << endl;
|
Chris@320
|
381 delete m_plugin;
|
Chris@320
|
382 delete m_descriptor;
|
Chris@320
|
383 }
|
Chris@320
|
384
|
Chris@320
|
385 DenseTimeValueModel *
|
Chris@350
|
386 FeatureExtractionModelTransformer::getConformingInput()
|
Chris@320
|
387 {
|
Chris@690
|
388 // SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
|
Chris@408
|
389
|
Chris@320
|
390 DenseTimeValueModel *dtvm =
|
Chris@320
|
391 dynamic_cast<DenseTimeValueModel *>(getInputModel());
|
Chris@320
|
392 if (!dtvm) {
|
Chris@690
|
393 SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
|
Chris@320
|
394 }
|
Chris@320
|
395 return dtvm;
|
Chris@320
|
396 }
|
Chris@320
|
397
|
Chris@320
|
398 void
|
Chris@331
|
399 FeatureExtractionModelTransformer::run()
|
Chris@320
|
400 {
|
Chris@350
|
401 DenseTimeValueModel *input = getConformingInput();
|
Chris@320
|
402 if (!input) return;
|
Chris@320
|
403
|
Chris@320
|
404 if (!m_output) return;
|
Chris@320
|
405
|
Chris@497
|
406 while (!input->isReady() && !m_abandoned) {
|
Chris@690
|
407 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
|
Chris@497
|
408 usleep(500000);
|
Chris@320
|
409 }
|
Chris@497
|
410 if (m_abandoned) return;
|
Chris@320
|
411
|
Chris@350
|
412 size_t sampleRate = input->getSampleRate();
|
Chris@320
|
413
|
Chris@320
|
414 size_t channelCount = input->getChannelCount();
|
Chris@320
|
415 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
416 channelCount = 1;
|
Chris@320
|
417 }
|
Chris@320
|
418
|
Chris@320
|
419 float **buffers = new float*[channelCount];
|
Chris@320
|
420 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
421 buffers[ch] = new float[m_transform.getBlockSize() + 2];
|
Chris@320
|
422 }
|
Chris@320
|
423
|
Chris@350
|
424 size_t stepSize = m_transform.getStepSize();
|
Chris@350
|
425 size_t blockSize = m_transform.getBlockSize();
|
Chris@350
|
426
|
Chris@320
|
427 bool frequencyDomain = (m_plugin->getInputDomain() ==
|
Chris@320
|
428 Vamp::Plugin::FrequencyDomain);
|
Chris@320
|
429 std::vector<FFTModel *> fftModels;
|
Chris@320
|
430
|
Chris@320
|
431 if (frequencyDomain) {
|
Chris@320
|
432 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
433 FFTModel *model = new FFTModel
|
Chris@350
|
434 (getConformingInput(),
|
Chris@350
|
435 channelCount == 1 ? m_input.getChannel() : ch,
|
Chris@350
|
436 m_transform.getWindowType(),
|
Chris@350
|
437 blockSize,
|
Chris@350
|
438 stepSize,
|
Chris@350
|
439 blockSize,
|
Chris@334
|
440 false,
|
Chris@334
|
441 StorageAdviser::PrecisionCritical);
|
Chris@320
|
442 if (!model->isOK()) {
|
Chris@320
|
443 delete model;
|
Chris@320
|
444 setCompletion(100);
|
Chris@387
|
445 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
|
Chris@387
|
446 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
|
Chris@320
|
447 }
|
Chris@320
|
448 model->resume();
|
Chris@320
|
449 fftModels.push_back(model);
|
Chris@320
|
450 }
|
Chris@320
|
451 }
|
Chris@320
|
452
|
Chris@350
|
453 long startFrame = m_input.getModel()->getStartFrame();
|
Chris@350
|
454 long endFrame = m_input.getModel()->getEndFrame();
|
Chris@320
|
455
|
Chris@350
|
456 RealTime contextStartRT = m_transform.getStartTime();
|
Chris@350
|
457 RealTime contextDurationRT = m_transform.getDuration();
|
Chris@350
|
458
|
Chris@350
|
459 long contextStart =
|
Chris@350
|
460 RealTime::realTime2Frame(contextStartRT, sampleRate);
|
Chris@350
|
461
|
Chris@350
|
462 long contextDuration =
|
Chris@350
|
463 RealTime::realTime2Frame(contextDurationRT, sampleRate);
|
Chris@320
|
464
|
Chris@320
|
465 if (contextStart == 0 || contextStart < startFrame) {
|
Chris@320
|
466 contextStart = startFrame;
|
Chris@320
|
467 }
|
Chris@320
|
468
|
Chris@320
|
469 if (contextDuration == 0) {
|
Chris@320
|
470 contextDuration = endFrame - contextStart;
|
Chris@320
|
471 }
|
Chris@320
|
472 if (contextStart + contextDuration > endFrame) {
|
Chris@320
|
473 contextDuration = endFrame - contextStart;
|
Chris@320
|
474 }
|
Chris@320
|
475
|
Chris@320
|
476 long blockFrame = contextStart;
|
Chris@320
|
477
|
Chris@320
|
478 long prevCompletion = 0;
|
Chris@320
|
479
|
Chris@320
|
480 setCompletion(0);
|
Chris@320
|
481
|
Chris@556
|
482 float *reals = 0;
|
Chris@556
|
483 float *imaginaries = 0;
|
Chris@556
|
484 if (frequencyDomain) {
|
Chris@556
|
485 reals = new float[blockSize/2 + 1];
|
Chris@556
|
486 imaginaries = new float[blockSize/2 + 1];
|
Chris@556
|
487 }
|
Chris@556
|
488
|
Chris@678
|
489 QString error = "";
|
Chris@678
|
490
|
Chris@320
|
491 while (!m_abandoned) {
|
Chris@320
|
492
|
Chris@320
|
493 if (frequencyDomain) {
|
Chris@350
|
494 if (blockFrame - int(blockSize)/2 >
|
Chris@320
|
495 contextStart + contextDuration) break;
|
Chris@320
|
496 } else {
|
Chris@320
|
497 if (blockFrame >=
|
Chris@320
|
498 contextStart + contextDuration) break;
|
Chris@320
|
499 }
|
Chris@320
|
500
|
Chris@690
|
501 // SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
|
Chris@320
|
502 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
|
Chris@687
|
503 // << blockSize << endl;
|
Chris@320
|
504
|
Chris@320
|
505 long completion =
|
Chris@350
|
506 (((blockFrame - contextStart) / stepSize) * 99) /
|
Chris@557
|
507 (contextDuration / stepSize + 1);
|
Chris@320
|
508
|
Chris@350
|
509 // channelCount is either m_input.getModel()->channelCount or 1
|
Chris@320
|
510
|
Chris@363
|
511 if (frequencyDomain) {
|
Chris@363
|
512 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
513 int column = (blockFrame - startFrame) / stepSize;
|
Chris@556
|
514 fftModels[ch]->getValuesAt(column, reals, imaginaries);
|
Chris@350
|
515 for (size_t i = 0; i <= blockSize/2; ++i) {
|
Chris@556
|
516 buffers[ch][i*2] = reals[i];
|
Chris@556
|
517 buffers[ch][i*2+1] = imaginaries[i];
|
Chris@320
|
518 }
|
Chris@678
|
519 error = fftModels[ch]->getError();
|
Chris@678
|
520 if (error != "") {
|
Chris@686
|
521 std::cerr << "FeatureExtractionModelTransformer::run: Abandoning, error is " << error << std::endl;
|
Chris@678
|
522 m_abandoned = true;
|
Chris@678
|
523 m_message = error;
|
Chris@678
|
524 }
|
Chris@363
|
525 }
|
Chris@363
|
526 } else {
|
Chris@363
|
527 getFrames(channelCount, blockFrame, blockSize, buffers);
|
Chris@320
|
528 }
|
Chris@320
|
529
|
Chris@497
|
530 if (m_abandoned) break;
|
Chris@497
|
531
|
Chris@320
|
532 Vamp::Plugin::FeatureSet features = m_plugin->process
|
Chris@320
|
533 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
|
Chris@320
|
534
|
Chris@497
|
535 if (m_abandoned) break;
|
Chris@497
|
536
|
Chris@778
|
537 for (size_t fi = 0; fi < features[m_outputNo].size(); ++fi) {
|
Chris@778
|
538 Vamp::Plugin::Feature feature = features[m_outputNo][fi];
|
Chris@320
|
539 addFeature(blockFrame, feature);
|
Chris@320
|
540 }
|
Chris@320
|
541
|
Chris@320
|
542 if (blockFrame == contextStart || completion > prevCompletion) {
|
Chris@320
|
543 setCompletion(completion);
|
Chris@320
|
544 prevCompletion = completion;
|
Chris@320
|
545 }
|
Chris@320
|
546
|
Chris@350
|
547 blockFrame += stepSize;
|
Chris@320
|
548 }
|
Chris@320
|
549
|
Chris@497
|
550 if (!m_abandoned) {
|
Chris@497
|
551 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
|
Chris@320
|
552
|
Chris@778
|
553 for (size_t fi = 0; fi < features[m_outputNo].size(); ++fi) {
|
Chris@778
|
554 Vamp::Plugin::Feature feature = features[m_outputNo][fi];
|
Chris@497
|
555 addFeature(blockFrame, feature);
|
Chris@497
|
556 }
|
Chris@497
|
557 }
|
Chris@320
|
558
|
Chris@497
|
559 setCompletion(100);
|
Chris@320
|
560
|
Chris@320
|
561 if (frequencyDomain) {
|
Chris@320
|
562 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
563 delete fftModels[ch];
|
Chris@320
|
564 }
|
Chris@556
|
565 delete[] reals;
|
Chris@556
|
566 delete[] imaginaries;
|
Chris@320
|
567 }
|
Chris@320
|
568 }
|
Chris@320
|
569
|
Chris@320
|
570 void
|
Chris@363
|
571 FeatureExtractionModelTransformer::getFrames(int channelCount,
|
Chris@363
|
572 long startFrame, long size,
|
Chris@363
|
573 float **buffers)
|
Chris@320
|
574 {
|
Chris@320
|
575 long offset = 0;
|
Chris@320
|
576
|
Chris@320
|
577 if (startFrame < 0) {
|
Chris@363
|
578 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
579 for (int i = 0; i < size && startFrame + i < 0; ++i) {
|
Chris@363
|
580 buffers[c][i] = 0.0f;
|
Chris@363
|
581 }
|
Chris@320
|
582 }
|
Chris@320
|
583 offset = -startFrame;
|
Chris@320
|
584 size -= offset;
|
Chris@320
|
585 if (size <= 0) return;
|
Chris@320
|
586 startFrame = 0;
|
Chris@320
|
587 }
|
Chris@320
|
588
|
Chris@350
|
589 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
590 if (!input) return;
|
Chris@363
|
591
|
Chris@363
|
592 long got = 0;
|
Chris@350
|
593
|
Chris@363
|
594 if (channelCount == 1) {
|
Chris@363
|
595
|
Chris@363
|
596 got = input->getData(m_input.getChannel(), startFrame, size,
|
Chris@363
|
597 buffers[0] + offset);
|
Chris@363
|
598
|
Chris@363
|
599 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
|
Chris@363
|
600 // use mean instead of sum, as plugin input
|
Chris@363
|
601 float cc = float(input->getChannelCount());
|
Chris@363
|
602 for (long i = 0; i < size; ++i) {
|
Chris@363
|
603 buffers[0][i + offset] /= cc;
|
Chris@363
|
604 }
|
Chris@363
|
605 }
|
Chris@363
|
606
|
Chris@363
|
607 } else {
|
Chris@363
|
608
|
Chris@363
|
609 float **writebuf = buffers;
|
Chris@363
|
610 if (offset > 0) {
|
Chris@363
|
611 writebuf = new float *[channelCount];
|
Chris@363
|
612 for (int i = 0; i < channelCount; ++i) {
|
Chris@363
|
613 writebuf[i] = buffers[i] + offset;
|
Chris@363
|
614 }
|
Chris@363
|
615 }
|
Chris@363
|
616
|
Chris@363
|
617 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
|
Chris@363
|
618
|
Chris@363
|
619 if (writebuf != buffers) delete[] writebuf;
|
Chris@363
|
620 }
|
Chris@320
|
621
|
Chris@320
|
622 while (got < size) {
|
Chris@363
|
623 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
624 buffers[c][got + offset] = 0.0;
|
Chris@363
|
625 }
|
Chris@320
|
626 ++got;
|
Chris@320
|
627 }
|
Chris@320
|
628 }
|
Chris@320
|
629
|
Chris@320
|
630 void
|
Chris@331
|
631 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
|
Chris@320
|
632 const Vamp::Plugin::Feature &feature)
|
Chris@320
|
633 {
|
Chris@350
|
634 size_t inputRate = m_input.getModel()->getSampleRate();
|
Chris@320
|
635
|
Chris@712
|
636 // std::cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
|
Chris@712
|
637 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
|
Chris@712
|
638 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
|
Chris@712
|
639 // << feature.hasDuration << ", duration = " << feature.duration
|
Chris@712
|
640 // << std::endl;
|
Chris@320
|
641
|
Chris@320
|
642 int binCount = 1;
|
Chris@320
|
643 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
644 binCount = m_descriptor->binCount;
|
Chris@320
|
645 }
|
Chris@320
|
646
|
Chris@320
|
647 size_t frame = blockFrame;
|
Chris@320
|
648
|
Chris@320
|
649 if (m_descriptor->sampleType ==
|
Chris@320
|
650 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@320
|
651
|
Chris@320
|
652 if (!feature.hasTimestamp) {
|
Chris@320
|
653 std::cerr
|
Chris@331
|
654 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
|
Chris@320
|
655 << "Feature has variable sample rate but no timestamp!"
|
Chris@320
|
656 << std::endl;
|
Chris@320
|
657 return;
|
Chris@320
|
658 } else {
|
Chris@320
|
659 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
|
Chris@320
|
660 }
|
Chris@320
|
661
|
Chris@320
|
662 } else if (m_descriptor->sampleType ==
|
Chris@320
|
663 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
|
Chris@320
|
664
|
Chris@320
|
665 if (feature.hasTimestamp) {
|
Chris@320
|
666 //!!! warning: sampleRate may be non-integral
|
Chris@320
|
667 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
|
Chris@451
|
668 //!!! see comment above when setting up modelResolution and modelRate
|
Chris@451
|
669 // lrintf(m_descriptor->sampleRate));
|
Chris@451
|
670 inputRate);
|
Chris@320
|
671 } else {
|
Chris@320
|
672 frame = m_output->getEndFrame();
|
Chris@320
|
673 }
|
Chris@777
|
674
|
Chris@777
|
675 // std::cerr << "Feature hasTimestamp = " << feature.hasTimestamp << ", timestamp = " << feature.timestamp << ", frame works out to " << frame << std::endl;
|
Chris@777
|
676
|
Chris@320
|
677 }
|
Chris@320
|
678
|
Chris@441
|
679 // Rather than repeat the complicated tests from the constructor
|
Chris@441
|
680 // to determine what sort of model we must be adding the features
|
Chris@441
|
681 // to, we instead test what sort of model the constructor decided
|
Chris@441
|
682 // to create.
|
Chris@320
|
683
|
Chris@441
|
684 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@441
|
685
|
Chris@441
|
686 SparseOneDimensionalModel *model =
|
Chris@350
|
687 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
688 if (!model) return;
|
Chris@350
|
689
|
Chris@441
|
690 model->addPoint(SparseOneDimensionalModel::Point
|
Chris@441
|
691 (frame, feature.label.c_str()));
|
Chris@320
|
692
|
Chris@441
|
693 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
694
|
Chris@350
|
695 SparseTimeValueModel *model =
|
Chris@350
|
696 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
697 if (!model) return;
|
Chris@350
|
698
|
Chris@454
|
699 for (int i = 0; i < feature.values.size(); ++i) {
|
Chris@454
|
700
|
Chris@454
|
701 float value = feature.values[i];
|
Chris@454
|
702
|
Chris@454
|
703 QString label = feature.label.c_str();
|
Chris@454
|
704 if (feature.values.size() > 1) {
|
Chris@454
|
705 label = QString("[%1] %2").arg(i+1).arg(label);
|
Chris@454
|
706 }
|
Chris@454
|
707
|
Chris@777
|
708 std::cerr << "Adding point at " << frame << " with value " << value << " and label " << label << std::endl;
|
Chris@777
|
709
|
Chris@454
|
710 model->addPoint(SparseTimeValueModel::Point(frame, value, label));
|
Chris@454
|
711 }
|
Chris@320
|
712
|
Chris@441
|
713 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
|
Chris@320
|
714
|
Chris@441
|
715 int index = 0;
|
Chris@441
|
716
|
Chris@441
|
717 float value = 0.0;
|
Chris@441
|
718 if (feature.values.size() > index) {
|
Chris@441
|
719 value = feature.values[index++];
|
Chris@441
|
720 }
|
Chris@320
|
721
|
Chris@320
|
722 float duration = 1;
|
Chris@441
|
723 if (feature.hasDuration) {
|
Chris@441
|
724 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
|
Chris@441
|
725 } else {
|
Chris@441
|
726 if (feature.values.size() > index) {
|
Chris@441
|
727 duration = feature.values[index++];
|
Chris@441
|
728 }
|
Chris@441
|
729 }
|
Chris@320
|
730
|
Chris@441
|
731 if (isOutput<NoteModel>()) {
|
Chris@320
|
732
|
Chris@441
|
733 float velocity = 100;
|
Chris@441
|
734 if (feature.values.size() > index) {
|
Chris@441
|
735 velocity = feature.values[index++];
|
Chris@441
|
736 }
|
Chris@441
|
737 if (velocity < 0) velocity = 127;
|
Chris@441
|
738 if (velocity > 127) velocity = 127;
|
Chris@320
|
739
|
Chris@441
|
740 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@441
|
741 if (!model) return;
|
Chris@441
|
742 model->addPoint(NoteModel::Point(frame, value, // value is pitch
|
Chris@441
|
743 lrintf(duration),
|
Chris@441
|
744 velocity / 127.f,
|
Chris@441
|
745 feature.label.c_str()));
|
Chris@441
|
746 } else {
|
Chris@441
|
747 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@454
|
748 if (!model) return;
|
Chris@454
|
749
|
Chris@474
|
750 if (feature.hasDuration && !feature.values.empty()) {
|
Chris@454
|
751
|
Chris@454
|
752 for (int i = 0; i < feature.values.size(); ++i) {
|
Chris@454
|
753
|
Chris@454
|
754 float value = feature.values[i];
|
Chris@454
|
755
|
Chris@454
|
756 QString label = feature.label.c_str();
|
Chris@454
|
757 if (feature.values.size() > 1) {
|
Chris@454
|
758 label = QString("[%1] %2").arg(i+1).arg(label);
|
Chris@454
|
759 }
|
Chris@454
|
760
|
Chris@454
|
761 model->addPoint(RegionModel::Point(frame, value,
|
Chris@454
|
762 lrintf(duration),
|
Chris@454
|
763 label));
|
Chris@454
|
764 }
|
Chris@454
|
765 } else {
|
Chris@454
|
766
|
Chris@441
|
767 model->addPoint(RegionModel::Point(frame, value,
|
Chris@441
|
768 lrintf(duration),
|
Chris@441
|
769 feature.label.c_str()));
|
Chris@454
|
770 }
|
Chris@441
|
771 }
|
Chris@320
|
772
|
Chris@441
|
773 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
774
|
Chris@533
|
775 DenseThreeDimensionalModel::Column values =
|
Chris@533
|
776 DenseThreeDimensionalModel::Column::fromStdVector(feature.values);
|
Chris@320
|
777
|
Chris@320
|
778 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
779 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
780 if (!model) return;
|
Chris@320
|
781
|
Chris@320
|
782 model->setColumn(frame / model->getResolution(), values);
|
Chris@441
|
783
|
Chris@441
|
784 } else {
|
Chris@690
|
785 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
|
Chris@320
|
786 }
|
Chris@320
|
787 }
|
Chris@320
|
788
|
Chris@320
|
789 void
|
Chris@331
|
790 FeatureExtractionModelTransformer::setCompletion(int completion)
|
Chris@320
|
791 {
|
Chris@320
|
792 int binCount = 1;
|
Chris@320
|
793 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
794 binCount = m_descriptor->binCount;
|
Chris@320
|
795 }
|
Chris@320
|
796
|
Chris@690
|
797 // SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
|
Chris@687
|
798 // << completion << ")" << endl;
|
Chris@320
|
799
|
Chris@441
|
800 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@320
|
801
|
Chris@350
|
802 SparseOneDimensionalModel *model =
|
Chris@350
|
803 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
804 if (!model) return;
|
Chris@441
|
805 model->setCompletion(completion, true);
|
Chris@320
|
806
|
Chris@441
|
807 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
808
|
Chris@350
|
809 SparseTimeValueModel *model =
|
Chris@350
|
810 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
811 if (!model) return;
|
Chris@441
|
812 model->setCompletion(completion, true);
|
Chris@320
|
813
|
Chris@441
|
814 } else if (isOutput<NoteModel>()) {
|
Chris@320
|
815
|
Chris@441
|
816 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@320
|
817 if (!model) return;
|
Chris@441
|
818 model->setCompletion(completion, true);
|
Chris@320
|
819
|
Chris@441
|
820 } else if (isOutput<RegionModel>()) {
|
Chris@441
|
821
|
Chris@441
|
822 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@441
|
823 if (!model) return;
|
Chris@441
|
824 model->setCompletion(completion, true);
|
Chris@441
|
825
|
Chris@441
|
826 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
827
|
Chris@320
|
828 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
829 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
830 if (!model) return;
|
Chris@350
|
831 model->setCompletion(completion, true); //!!!m_context.updates);
|
Chris@320
|
832 }
|
Chris@320
|
833 }
|
Chris@320
|
834
|