Chris@320
|
1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
|
Chris@320
|
2
|
Chris@320
|
3 /*
|
Chris@320
|
4 Sonic Visualiser
|
Chris@320
|
5 An audio file viewer and annotation editor.
|
Chris@320
|
6 Centre for Digital Music, Queen Mary, University of London.
|
Chris@320
|
7 This file copyright 2006 Chris Cannam and QMUL.
|
Chris@320
|
8
|
Chris@320
|
9 This program is free software; you can redistribute it and/or
|
Chris@320
|
10 modify it under the terms of the GNU General Public License as
|
Chris@320
|
11 published by the Free Software Foundation; either version 2 of the
|
Chris@320
|
12 License, or (at your option) any later version. See the file
|
Chris@320
|
13 COPYING included with this distribution for more information.
|
Chris@320
|
14 */
|
Chris@320
|
15
|
Chris@331
|
16 #include "FeatureExtractionModelTransformer.h"
|
Chris@320
|
17
|
Chris@320
|
18 #include "plugin/FeatureExtractionPluginFactory.h"
|
Chris@320
|
19 #include "plugin/PluginXml.h"
|
Chris@320
|
20 #include "vamp-sdk/Plugin.h"
|
Chris@320
|
21
|
Chris@320
|
22 #include "data/model/Model.h"
|
Chris@320
|
23 #include "base/Window.h"
|
Chris@387
|
24 #include "base/Exceptions.h"
|
Chris@320
|
25 #include "data/model/SparseOneDimensionalModel.h"
|
Chris@320
|
26 #include "data/model/SparseTimeValueModel.h"
|
Chris@320
|
27 #include "data/model/EditableDenseThreeDimensionalModel.h"
|
Chris@320
|
28 #include "data/model/DenseTimeValueModel.h"
|
Chris@320
|
29 #include "data/model/NoteModel.h"
|
Chris@441
|
30 #include "data/model/RegionModel.h"
|
Chris@320
|
31 #include "data/model/FFTModel.h"
|
Chris@320
|
32 #include "data/model/WaveFileModel.h"
|
Chris@320
|
33
|
Chris@350
|
34 #include "TransformFactory.h"
|
Chris@350
|
35
|
Chris@320
|
36 #include <iostream>
|
Chris@320
|
37
|
Chris@350
|
38 FeatureExtractionModelTransformer::FeatureExtractionModelTransformer(Input in,
|
Chris@350
|
39 const Transform &transform) :
|
Chris@350
|
40 ModelTransformer(in, transform),
|
Chris@320
|
41 m_plugin(0),
|
Chris@320
|
42 m_descriptor(0),
|
Chris@320
|
43 m_outputFeatureNo(0)
|
Chris@320
|
44 {
|
Chris@350
|
45 // std::cerr << "FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " << pluginId.toStdString() << ", outputName " << m_transform.getOutput().toStdString() << std::endl;
|
Chris@350
|
46
|
Chris@350
|
47 QString pluginId = transform.getPluginIdentifier();
|
Chris@320
|
48
|
Chris@320
|
49 FeatureExtractionPluginFactory *factory =
|
Chris@320
|
50 FeatureExtractionPluginFactory::instanceFor(pluginId);
|
Chris@320
|
51
|
Chris@320
|
52 if (!factory) {
|
Chris@361
|
53 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
|
Chris@320
|
54 return;
|
Chris@320
|
55 }
|
Chris@320
|
56
|
Chris@350
|
57 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
58 if (!input) {
|
Chris@361
|
59 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
|
Chris@350
|
60 return;
|
Chris@350
|
61 }
|
Chris@320
|
62
|
Chris@350
|
63 m_plugin = factory->instantiatePlugin(pluginId, input->getSampleRate());
|
Chris@320
|
64 if (!m_plugin) {
|
Chris@361
|
65 m_message = tr("Failed to instantiate plugin \"%1\"").arg(pluginId);
|
Chris@320
|
66 return;
|
Chris@320
|
67 }
|
Chris@320
|
68
|
Chris@350
|
69 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@350
|
70 (m_transform, m_plugin);
|
Chris@343
|
71
|
Chris@350
|
72 TransformFactory::getInstance()->setPluginParameters
|
Chris@350
|
73 (m_transform, m_plugin);
|
Chris@320
|
74
|
Chris@320
|
75 size_t channelCount = input->getChannelCount();
|
Chris@320
|
76 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
77 channelCount = 1;
|
Chris@320
|
78 }
|
Chris@320
|
79 if (m_plugin->getMinChannelCount() > channelCount) {
|
Chris@361
|
80 m_message = tr("Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
|
Chris@361
|
81 .arg(pluginId)
|
Chris@361
|
82 .arg(m_plugin->getMinChannelCount())
|
Chris@361
|
83 .arg(m_plugin->getMaxChannelCount())
|
Chris@361
|
84 .arg(input->getChannelCount());
|
Chris@320
|
85 return;
|
Chris@320
|
86 }
|
Chris@320
|
87
|
Chris@320
|
88 std::cerr << "Initialising feature extraction plugin with channels = "
|
Chris@350
|
89 << channelCount << ", step = " << m_transform.getStepSize()
|
Chris@350
|
90 << ", block = " << m_transform.getBlockSize() << std::endl;
|
Chris@320
|
91
|
Chris@320
|
92 if (!m_plugin->initialise(channelCount,
|
Chris@350
|
93 m_transform.getStepSize(),
|
Chris@350
|
94 m_transform.getBlockSize())) {
|
Chris@361
|
95
|
Chris@361
|
96 size_t pstep = m_transform.getStepSize();
|
Chris@361
|
97 size_t pblock = m_transform.getBlockSize();
|
Chris@361
|
98
|
Chris@361
|
99 m_transform.setStepSize(0);
|
Chris@361
|
100 m_transform.setBlockSize(0);
|
Chris@361
|
101 TransformFactory::getInstance()->makeContextConsistentWithPlugin
|
Chris@361
|
102 (m_transform, m_plugin);
|
Chris@361
|
103
|
Chris@361
|
104 if (m_transform.getStepSize() != pstep ||
|
Chris@361
|
105 m_transform.getBlockSize() != pblock) {
|
Chris@361
|
106
|
Chris@361
|
107 if (!m_plugin->initialise(channelCount,
|
Chris@361
|
108 m_transform.getStepSize(),
|
Chris@361
|
109 m_transform.getBlockSize())) {
|
Chris@361
|
110
|
Chris@361
|
111 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
112 return;
|
Chris@361
|
113
|
Chris@361
|
114 } else {
|
Chris@361
|
115
|
Chris@361
|
116 m_message = tr("Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
|
Chris@361
|
117 .arg(pluginId)
|
Chris@361
|
118 .arg(pstep)
|
Chris@361
|
119 .arg(pblock)
|
Chris@361
|
120 .arg(m_transform.getStepSize())
|
Chris@361
|
121 .arg(m_transform.getBlockSize());
|
Chris@361
|
122 }
|
Chris@361
|
123
|
Chris@361
|
124 } else {
|
Chris@361
|
125
|
Chris@361
|
126 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
|
Chris@361
|
127 return;
|
Chris@361
|
128 }
|
Chris@320
|
129 }
|
Chris@320
|
130
|
Chris@366
|
131 if (m_transform.getPluginVersion() != "") {
|
Chris@366
|
132 QString pv = QString("%1").arg(m_plugin->getPluginVersion());
|
Chris@366
|
133 if (pv != m_transform.getPluginVersion()) {
|
Chris@366
|
134 QString vm = tr("Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
|
Chris@366
|
135 .arg(m_transform.getPluginVersion())
|
Chris@366
|
136 .arg(pluginId)
|
Chris@366
|
137 .arg(pv);
|
Chris@366
|
138 if (m_message != "") {
|
Chris@366
|
139 m_message = QString("%1; %2").arg(vm).arg(m_message);
|
Chris@366
|
140 } else {
|
Chris@366
|
141 m_message = vm;
|
Chris@366
|
142 }
|
Chris@366
|
143 }
|
Chris@366
|
144 }
|
Chris@366
|
145
|
Chris@320
|
146 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
|
Chris@320
|
147
|
Chris@320
|
148 if (outputs.empty()) {
|
Chris@361
|
149 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
|
Chris@320
|
150 return;
|
Chris@320
|
151 }
|
Chris@320
|
152
|
Chris@320
|
153 for (size_t i = 0; i < outputs.size(); ++i) {
|
Chris@429
|
154 // std::cerr << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput().toStdString() << "\"" << std::endl;
|
Chris@350
|
155 if (m_transform.getOutput() == "" ||
|
Chris@350
|
156 outputs[i].identifier == m_transform.getOutput().toStdString()) {
|
Chris@320
|
157 m_outputFeatureNo = i;
|
Chris@441
|
158 m_descriptor = new Vamp::Plugin::OutputDescriptor(outputs[i]);
|
Chris@320
|
159 break;
|
Chris@320
|
160 }
|
Chris@320
|
161 }
|
Chris@320
|
162
|
Chris@320
|
163 if (!m_descriptor) {
|
Chris@361
|
164 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
|
Chris@361
|
165 .arg(pluginId)
|
Chris@361
|
166 .arg(m_transform.getOutput());
|
Chris@320
|
167 return;
|
Chris@320
|
168 }
|
Chris@320
|
169
|
Chris@331
|
170 // std::cerr << "FeatureExtractionModelTransformer: output sample type "
|
Chris@320
|
171 // << m_descriptor->sampleType << std::endl;
|
Chris@320
|
172
|
Chris@320
|
173 int binCount = 1;
|
Chris@320
|
174 float minValue = 0.0, maxValue = 0.0;
|
Chris@320
|
175 bool haveExtents = false;
|
Chris@320
|
176
|
Chris@320
|
177 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
178 binCount = m_descriptor->binCount;
|
Chris@320
|
179 }
|
Chris@320
|
180
|
Chris@331
|
181 // std::cerr << "FeatureExtractionModelTransformer: output bin count "
|
Chris@320
|
182 // << binCount << std::endl;
|
Chris@320
|
183
|
Chris@320
|
184 if (binCount > 0 && m_descriptor->hasKnownExtents) {
|
Chris@320
|
185 minValue = m_descriptor->minValue;
|
Chris@320
|
186 maxValue = m_descriptor->maxValue;
|
Chris@320
|
187 haveExtents = true;
|
Chris@320
|
188 }
|
Chris@320
|
189
|
Chris@350
|
190 size_t modelRate = input->getSampleRate();
|
Chris@320
|
191 size_t modelResolution = 1;
|
Chris@320
|
192
|
Chris@320
|
193 switch (m_descriptor->sampleType) {
|
Chris@320
|
194
|
Chris@320
|
195 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
|
Chris@320
|
196 if (m_descriptor->sampleRate != 0.0) {
|
Chris@320
|
197 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
|
Chris@320
|
198 }
|
Chris@320
|
199 break;
|
Chris@320
|
200
|
Chris@320
|
201 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
|
Chris@350
|
202 modelResolution = m_transform.getStepSize();
|
Chris@320
|
203 break;
|
Chris@320
|
204
|
Chris@320
|
205 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
|
Chris@320
|
206 modelRate = size_t(m_descriptor->sampleRate + 0.001);
|
Chris@320
|
207 break;
|
Chris@320
|
208 }
|
Chris@320
|
209
|
Chris@441
|
210 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
|
Chris@441
|
211
|
Chris@441
|
212 if (binCount == 0 &&
|
Chris@441
|
213 (preDurationPlugin || !m_descriptor->hasDuration)) {
|
Chris@320
|
214
|
Chris@320
|
215 m_output = new SparseOneDimensionalModel(modelRate, modelResolution,
|
Chris@320
|
216 false);
|
Chris@320
|
217
|
Chris@441
|
218 } else if ((preDurationPlugin && binCount > 1 &&
|
Chris@441
|
219 (m_descriptor->sampleType ==
|
Chris@441
|
220 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
|
Chris@441
|
221 (!preDurationPlugin && m_descriptor->hasDuration)) {
|
Chris@441
|
222
|
Chris@441
|
223 // For plugins using the old v1 API without explicit duration,
|
Chris@441
|
224 // we treat anything that has multiple bins (i.e. that has the
|
Chris@441
|
225 // potential to have value and duration) and a variable sample
|
Chris@441
|
226 // rate as a note model, taking its values as pitch, duration
|
Chris@441
|
227 // and velocity (if present) respectively. This is the same
|
Chris@441
|
228 // behaviour as always applied by SV to these plugins in the
|
Chris@441
|
229 // past.
|
Chris@441
|
230
|
Chris@441
|
231 // For plugins with the newer API, we treat anything with
|
Chris@441
|
232 // duration as either a note model with pitch and velocity, or
|
Chris@441
|
233 // a region model.
|
Chris@441
|
234
|
Chris@441
|
235 // How do we know whether it's an interval or note model?
|
Chris@441
|
236 // What's the essential difference? Is a note model any
|
Chris@441
|
237 // interval model using a Hz or "MIDI pitch" scale? There
|
Chris@441
|
238 // isn't really a reliable test for "MIDI pitch"... Does a
|
Chris@441
|
239 // note model always have velocity? This is a good question
|
Chris@441
|
240 // to be addressed by accompanying RDF, but for the moment we
|
Chris@441
|
241 // will do the following...
|
Chris@441
|
242
|
Chris@441
|
243 bool isNoteModel = false;
|
Chris@441
|
244
|
Chris@441
|
245 // Regions have only value (and duration -- we can't extract a
|
Chris@441
|
246 // region model from an old-style plugin that doesn't support
|
Chris@441
|
247 // duration)
|
Chris@441
|
248 if (binCount > 1) isNoteModel = true;
|
Chris@441
|
249
|
Chris@441
|
250 // Regions do not have units of Hz (a sweeping assumption!)
|
Chris@441
|
251 if (m_descriptor->unit == "Hz") isNoteModel = true;
|
Chris@441
|
252
|
Chris@441
|
253 // If we had a "sparse 3D model", we would have the additional
|
Chris@441
|
254 // problem of determining whether to use that here (if bin
|
Chris@441
|
255 // count > 1). But we don't.
|
Chris@441
|
256
|
Chris@441
|
257 if (isNoteModel) {
|
Chris@441
|
258
|
Chris@441
|
259 NoteModel *model;
|
Chris@441
|
260 if (haveExtents) {
|
Chris@441
|
261 model = new NoteModel
|
Chris@441
|
262 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
263 } else {
|
Chris@441
|
264 model = new NoteModel
|
Chris@441
|
265 (modelRate, modelResolution, false);
|
Chris@441
|
266 }
|
Chris@441
|
267 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
268 m_output = model;
|
Chris@441
|
269
|
Chris@441
|
270 } else {
|
Chris@441
|
271
|
Chris@441
|
272 RegionModel *model;
|
Chris@441
|
273 if (haveExtents) {
|
Chris@441
|
274 model = new RegionModel
|
Chris@441
|
275 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@441
|
276 } else {
|
Chris@441
|
277 model = new RegionModel
|
Chris@441
|
278 (modelRate, modelResolution, false);
|
Chris@441
|
279 }
|
Chris@441
|
280 model->setScaleUnits(m_descriptor->unit.c_str());
|
Chris@441
|
281 m_output = model;
|
Chris@441
|
282 }
|
Chris@441
|
283
|
Chris@441
|
284 } else if (binCount == 1 ||
|
Chris@441
|
285 (m_descriptor->sampleType ==
|
Chris@441
|
286 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
|
Chris@441
|
287
|
Chris@441
|
288 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
289 // has only one value per result must be a sparse time value
|
Chris@441
|
290 // model.
|
Chris@441
|
291
|
Chris@441
|
292 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
293 // has a variable sample rate is also treated as a sparse time
|
Chris@441
|
294 // value model regardless of its bin count, because we lack a
|
Chris@441
|
295 // sparse 3D model.
|
Chris@320
|
296
|
Chris@320
|
297 SparseTimeValueModel *model;
|
Chris@320
|
298 if (haveExtents) {
|
Chris@320
|
299 model = new SparseTimeValueModel
|
Chris@320
|
300 (modelRate, modelResolution, minValue, maxValue, false);
|
Chris@320
|
301 } else {
|
Chris@320
|
302 model = new SparseTimeValueModel
|
Chris@320
|
303 (modelRate, modelResolution, false);
|
Chris@320
|
304 }
|
Chris@320
|
305 model->setScaleUnits(outputs[m_outputFeatureNo].unit.c_str());
|
Chris@320
|
306
|
Chris@320
|
307 m_output = model;
|
Chris@320
|
308
|
Chris@441
|
309 } else {
|
Chris@320
|
310
|
Chris@441
|
311 // Anything that is not a 1D, note, or interval model and that
|
Chris@441
|
312 // has a fixed sample rate and more than one value per result
|
Chris@441
|
313 // must be a dense 3D model.
|
Chris@320
|
314
|
Chris@320
|
315 EditableDenseThreeDimensionalModel *model =
|
Chris@320
|
316 new EditableDenseThreeDimensionalModel
|
Chris@320
|
317 (modelRate, modelResolution, binCount, false);
|
Chris@320
|
318
|
Chris@320
|
319 if (!m_descriptor->binNames.empty()) {
|
Chris@320
|
320 std::vector<QString> names;
|
Chris@320
|
321 for (size_t i = 0; i < m_descriptor->binNames.size(); ++i) {
|
Chris@320
|
322 names.push_back(m_descriptor->binNames[i].c_str());
|
Chris@320
|
323 }
|
Chris@320
|
324 model->setBinNames(names);
|
Chris@320
|
325 }
|
Chris@320
|
326
|
Chris@320
|
327 m_output = model;
|
Chris@320
|
328 }
|
Chris@333
|
329
|
Chris@350
|
330 if (m_output) m_output->setSourceModel(input);
|
Chris@320
|
331 }
|
Chris@320
|
332
|
Chris@331
|
333 FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()
|
Chris@320
|
334 {
|
Chris@436
|
335 // std::cerr << "FeatureExtractionModelTransformer::~FeatureExtractionModelTransformer()" << std::endl;
|
Chris@320
|
336 delete m_plugin;
|
Chris@320
|
337 delete m_descriptor;
|
Chris@320
|
338 }
|
Chris@320
|
339
|
Chris@320
|
340 DenseTimeValueModel *
|
Chris@350
|
341 FeatureExtractionModelTransformer::getConformingInput()
|
Chris@320
|
342 {
|
Chris@408
|
343 // std::cerr << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << std::endl;
|
Chris@408
|
344
|
Chris@320
|
345 DenseTimeValueModel *dtvm =
|
Chris@320
|
346 dynamic_cast<DenseTimeValueModel *>(getInputModel());
|
Chris@320
|
347 if (!dtvm) {
|
Chris@350
|
348 std::cerr << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
|
Chris@320
|
349 }
|
Chris@320
|
350 return dtvm;
|
Chris@320
|
351 }
|
Chris@320
|
352
|
Chris@320
|
353 void
|
Chris@331
|
354 FeatureExtractionModelTransformer::run()
|
Chris@320
|
355 {
|
Chris@350
|
356 DenseTimeValueModel *input = getConformingInput();
|
Chris@320
|
357 if (!input) return;
|
Chris@320
|
358
|
Chris@320
|
359 if (!m_output) return;
|
Chris@320
|
360
|
Chris@320
|
361 while (!input->isReady()) {
|
Chris@331
|
362 std::cerr << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << std::endl;
|
Chris@320
|
363 sleep(1);
|
Chris@320
|
364 }
|
Chris@320
|
365
|
Chris@350
|
366 size_t sampleRate = input->getSampleRate();
|
Chris@320
|
367
|
Chris@320
|
368 size_t channelCount = input->getChannelCount();
|
Chris@320
|
369 if (m_plugin->getMaxChannelCount() < channelCount) {
|
Chris@320
|
370 channelCount = 1;
|
Chris@320
|
371 }
|
Chris@320
|
372
|
Chris@320
|
373 float **buffers = new float*[channelCount];
|
Chris@320
|
374 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
375 buffers[ch] = new float[m_transform.getBlockSize() + 2];
|
Chris@320
|
376 }
|
Chris@320
|
377
|
Chris@350
|
378 size_t stepSize = m_transform.getStepSize();
|
Chris@350
|
379 size_t blockSize = m_transform.getBlockSize();
|
Chris@350
|
380
|
Chris@320
|
381 bool frequencyDomain = (m_plugin->getInputDomain() ==
|
Chris@320
|
382 Vamp::Plugin::FrequencyDomain);
|
Chris@320
|
383 std::vector<FFTModel *> fftModels;
|
Chris@320
|
384
|
Chris@320
|
385 if (frequencyDomain) {
|
Chris@320
|
386 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
387 FFTModel *model = new FFTModel
|
Chris@350
|
388 (getConformingInput(),
|
Chris@350
|
389 channelCount == 1 ? m_input.getChannel() : ch,
|
Chris@350
|
390 m_transform.getWindowType(),
|
Chris@350
|
391 blockSize,
|
Chris@350
|
392 stepSize,
|
Chris@350
|
393 blockSize,
|
Chris@334
|
394 false,
|
Chris@334
|
395 StorageAdviser::PrecisionCritical);
|
Chris@320
|
396 if (!model->isOK()) {
|
Chris@320
|
397 delete model;
|
Chris@320
|
398 setCompletion(100);
|
Chris@387
|
399 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
|
Chris@387
|
400 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer");
|
Chris@320
|
401 }
|
Chris@320
|
402 model->resume();
|
Chris@320
|
403 fftModels.push_back(model);
|
Chris@320
|
404 }
|
Chris@320
|
405 }
|
Chris@320
|
406
|
Chris@350
|
407 long startFrame = m_input.getModel()->getStartFrame();
|
Chris@350
|
408 long endFrame = m_input.getModel()->getEndFrame();
|
Chris@320
|
409
|
Chris@350
|
410 RealTime contextStartRT = m_transform.getStartTime();
|
Chris@350
|
411 RealTime contextDurationRT = m_transform.getDuration();
|
Chris@350
|
412
|
Chris@350
|
413 long contextStart =
|
Chris@350
|
414 RealTime::realTime2Frame(contextStartRT, sampleRate);
|
Chris@350
|
415
|
Chris@350
|
416 long contextDuration =
|
Chris@350
|
417 RealTime::realTime2Frame(contextDurationRT, sampleRate);
|
Chris@320
|
418
|
Chris@320
|
419 if (contextStart == 0 || contextStart < startFrame) {
|
Chris@320
|
420 contextStart = startFrame;
|
Chris@320
|
421 }
|
Chris@320
|
422
|
Chris@320
|
423 if (contextDuration == 0) {
|
Chris@320
|
424 contextDuration = endFrame - contextStart;
|
Chris@320
|
425 }
|
Chris@320
|
426 if (contextStart + contextDuration > endFrame) {
|
Chris@320
|
427 contextDuration = endFrame - contextStart;
|
Chris@320
|
428 }
|
Chris@320
|
429
|
Chris@320
|
430 long blockFrame = contextStart;
|
Chris@320
|
431
|
Chris@320
|
432 long prevCompletion = 0;
|
Chris@320
|
433
|
Chris@320
|
434 setCompletion(0);
|
Chris@320
|
435
|
Chris@320
|
436 while (!m_abandoned) {
|
Chris@320
|
437
|
Chris@320
|
438 if (frequencyDomain) {
|
Chris@350
|
439 if (blockFrame - int(blockSize)/2 >
|
Chris@320
|
440 contextStart + contextDuration) break;
|
Chris@320
|
441 } else {
|
Chris@320
|
442 if (blockFrame >=
|
Chris@320
|
443 contextStart + contextDuration) break;
|
Chris@320
|
444 }
|
Chris@320
|
445
|
Chris@331
|
446 // std::cerr << "FeatureExtractionModelTransformer::run: blockFrame "
|
Chris@320
|
447 // << blockFrame << ", endFrame " << endFrame << ", blockSize "
|
Chris@350
|
448 // << blockSize << std::endl;
|
Chris@320
|
449
|
Chris@320
|
450 long completion =
|
Chris@350
|
451 (((blockFrame - contextStart) / stepSize) * 99) /
|
Chris@350
|
452 (contextDuration / stepSize);
|
Chris@320
|
453
|
Chris@350
|
454 // channelCount is either m_input.getModel()->channelCount or 1
|
Chris@320
|
455
|
Chris@363
|
456 if (frequencyDomain) {
|
Chris@363
|
457 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@350
|
458 int column = (blockFrame - startFrame) / stepSize;
|
Chris@350
|
459 for (size_t i = 0; i <= blockSize/2; ++i) {
|
Chris@320
|
460 fftModels[ch]->getValuesAt
|
Chris@320
|
461 (column, i, buffers[ch][i*2], buffers[ch][i*2+1]);
|
Chris@320
|
462 }
|
Chris@363
|
463 }
|
Chris@363
|
464 } else {
|
Chris@363
|
465 getFrames(channelCount, blockFrame, blockSize, buffers);
|
Chris@320
|
466 }
|
Chris@320
|
467
|
Chris@320
|
468 Vamp::Plugin::FeatureSet features = m_plugin->process
|
Chris@320
|
469 (buffers, Vamp::RealTime::frame2RealTime(blockFrame, sampleRate));
|
Chris@320
|
470
|
Chris@320
|
471 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@320
|
472 Vamp::Plugin::Feature feature =
|
Chris@320
|
473 features[m_outputFeatureNo][fi];
|
Chris@320
|
474 addFeature(blockFrame, feature);
|
Chris@320
|
475 }
|
Chris@320
|
476
|
Chris@320
|
477 if (blockFrame == contextStart || completion > prevCompletion) {
|
Chris@320
|
478 setCompletion(completion);
|
Chris@320
|
479 prevCompletion = completion;
|
Chris@320
|
480 }
|
Chris@320
|
481
|
Chris@350
|
482 blockFrame += stepSize;
|
Chris@320
|
483 }
|
Chris@320
|
484
|
Chris@320
|
485 if (m_abandoned) return;
|
Chris@320
|
486
|
Chris@320
|
487 Vamp::Plugin::FeatureSet features = m_plugin->getRemainingFeatures();
|
Chris@320
|
488
|
Chris@320
|
489 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
|
Chris@320
|
490 Vamp::Plugin::Feature feature =
|
Chris@320
|
491 features[m_outputFeatureNo][fi];
|
Chris@320
|
492 addFeature(blockFrame, feature);
|
Chris@320
|
493 }
|
Chris@320
|
494
|
Chris@320
|
495 if (frequencyDomain) {
|
Chris@320
|
496 for (size_t ch = 0; ch < channelCount; ++ch) {
|
Chris@320
|
497 delete fftModels[ch];
|
Chris@320
|
498 }
|
Chris@320
|
499 }
|
Chris@320
|
500
|
Chris@320
|
501 setCompletion(100);
|
Chris@320
|
502 }
|
Chris@320
|
503
|
Chris@320
|
504 void
|
Chris@363
|
505 FeatureExtractionModelTransformer::getFrames(int channelCount,
|
Chris@363
|
506 long startFrame, long size,
|
Chris@363
|
507 float **buffers)
|
Chris@320
|
508 {
|
Chris@320
|
509 long offset = 0;
|
Chris@320
|
510
|
Chris@320
|
511 if (startFrame < 0) {
|
Chris@363
|
512 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
513 for (int i = 0; i < size && startFrame + i < 0; ++i) {
|
Chris@363
|
514 buffers[c][i] = 0.0f;
|
Chris@363
|
515 }
|
Chris@320
|
516 }
|
Chris@320
|
517 offset = -startFrame;
|
Chris@320
|
518 size -= offset;
|
Chris@320
|
519 if (size <= 0) return;
|
Chris@320
|
520 startFrame = 0;
|
Chris@320
|
521 }
|
Chris@320
|
522
|
Chris@350
|
523 DenseTimeValueModel *input = getConformingInput();
|
Chris@350
|
524 if (!input) return;
|
Chris@363
|
525
|
Chris@363
|
526 long got = 0;
|
Chris@350
|
527
|
Chris@363
|
528 if (channelCount == 1) {
|
Chris@363
|
529
|
Chris@363
|
530 got = input->getData(m_input.getChannel(), startFrame, size,
|
Chris@363
|
531 buffers[0] + offset);
|
Chris@363
|
532
|
Chris@363
|
533 if (m_input.getChannel() == -1 && input->getChannelCount() > 1) {
|
Chris@363
|
534 // use mean instead of sum, as plugin input
|
Chris@363
|
535 float cc = float(input->getChannelCount());
|
Chris@363
|
536 for (long i = 0; i < size; ++i) {
|
Chris@363
|
537 buffers[0][i + offset] /= cc;
|
Chris@363
|
538 }
|
Chris@363
|
539 }
|
Chris@363
|
540
|
Chris@363
|
541 } else {
|
Chris@363
|
542
|
Chris@363
|
543 float **writebuf = buffers;
|
Chris@363
|
544 if (offset > 0) {
|
Chris@363
|
545 writebuf = new float *[channelCount];
|
Chris@363
|
546 for (int i = 0; i < channelCount; ++i) {
|
Chris@363
|
547 writebuf[i] = buffers[i] + offset;
|
Chris@363
|
548 }
|
Chris@363
|
549 }
|
Chris@363
|
550
|
Chris@363
|
551 got = input->getData(0, channelCount-1, startFrame, size, writebuf);
|
Chris@363
|
552
|
Chris@363
|
553 if (writebuf != buffers) delete[] writebuf;
|
Chris@363
|
554 }
|
Chris@320
|
555
|
Chris@320
|
556 while (got < size) {
|
Chris@363
|
557 for (int c = 0; c < channelCount; ++c) {
|
Chris@363
|
558 buffers[c][got + offset] = 0.0;
|
Chris@363
|
559 }
|
Chris@320
|
560 ++got;
|
Chris@320
|
561 }
|
Chris@320
|
562 }
|
Chris@320
|
563
|
Chris@320
|
564 void
|
Chris@331
|
565 FeatureExtractionModelTransformer::addFeature(size_t blockFrame,
|
Chris@320
|
566 const Vamp::Plugin::Feature &feature)
|
Chris@320
|
567 {
|
Chris@350
|
568 size_t inputRate = m_input.getModel()->getSampleRate();
|
Chris@320
|
569
|
Chris@331
|
570 // std::cerr << "FeatureExtractionModelTransformer::addFeature("
|
Chris@320
|
571 // << blockFrame << ")" << std::endl;
|
Chris@320
|
572
|
Chris@320
|
573 int binCount = 1;
|
Chris@320
|
574 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
575 binCount = m_descriptor->binCount;
|
Chris@320
|
576 }
|
Chris@320
|
577
|
Chris@320
|
578 size_t frame = blockFrame;
|
Chris@320
|
579
|
Chris@320
|
580 if (m_descriptor->sampleType ==
|
Chris@320
|
581 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
|
Chris@320
|
582
|
Chris@320
|
583 if (!feature.hasTimestamp) {
|
Chris@320
|
584 std::cerr
|
Chris@331
|
585 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
|
Chris@320
|
586 << "Feature has variable sample rate but no timestamp!"
|
Chris@320
|
587 << std::endl;
|
Chris@320
|
588 return;
|
Chris@320
|
589 } else {
|
Chris@320
|
590 frame = Vamp::RealTime::realTime2Frame(feature.timestamp, inputRate);
|
Chris@320
|
591 }
|
Chris@320
|
592
|
Chris@320
|
593 } else if (m_descriptor->sampleType ==
|
Chris@320
|
594 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
|
Chris@320
|
595
|
Chris@320
|
596 if (feature.hasTimestamp) {
|
Chris@320
|
597 //!!! warning: sampleRate may be non-integral
|
Chris@320
|
598 frame = Vamp::RealTime::realTime2Frame(feature.timestamp,
|
Chris@320
|
599 lrintf(m_descriptor->sampleRate));
|
Chris@320
|
600 } else {
|
Chris@320
|
601 frame = m_output->getEndFrame();
|
Chris@320
|
602 }
|
Chris@320
|
603 }
|
Chris@320
|
604
|
Chris@441
|
605 // Rather than repeat the complicated tests from the constructor
|
Chris@441
|
606 // to determine what sort of model we must be adding the features
|
Chris@441
|
607 // to, we instead test what sort of model the constructor decided
|
Chris@441
|
608 // to create.
|
Chris@320
|
609
|
Chris@441
|
610 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@441
|
611
|
Chris@441
|
612 SparseOneDimensionalModel *model =
|
Chris@350
|
613 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
614 if (!model) return;
|
Chris@350
|
615
|
Chris@441
|
616 model->addPoint(SparseOneDimensionalModel::Point
|
Chris@441
|
617 (frame, feature.label.c_str()));
|
Chris@320
|
618
|
Chris@441
|
619 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
620
|
Chris@320
|
621 float value = 0.0;
|
Chris@320
|
622 if (feature.values.size() > 0) value = feature.values[0];
|
Chris@320
|
623
|
Chris@350
|
624 SparseTimeValueModel *model =
|
Chris@350
|
625 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
626 if (!model) return;
|
Chris@350
|
627
|
Chris@441
|
628 model->addPoint(SparseTimeValueModel::Point
|
Chris@441
|
629 (frame, value, feature.label.c_str()));
|
Chris@320
|
630
|
Chris@441
|
631 } else if (isOutput<NoteModel>() || isOutput<RegionModel>()) {
|
Chris@320
|
632
|
Chris@441
|
633 int index = 0;
|
Chris@441
|
634
|
Chris@441
|
635 float value = 0.0;
|
Chris@441
|
636 if (feature.values.size() > index) {
|
Chris@441
|
637 value = feature.values[index++];
|
Chris@441
|
638 }
|
Chris@320
|
639
|
Chris@320
|
640 float duration = 1;
|
Chris@441
|
641 if (feature.hasDuration) {
|
Chris@441
|
642 duration = Vamp::RealTime::realTime2Frame(feature.duration, inputRate);
|
Chris@441
|
643 } else {
|
Chris@441
|
644 if (feature.values.size() > index) {
|
Chris@441
|
645 duration = feature.values[index++];
|
Chris@441
|
646 }
|
Chris@441
|
647 }
|
Chris@320
|
648
|
Chris@441
|
649 if (isOutput<NoteModel>()) {
|
Chris@320
|
650
|
Chris@441
|
651 float velocity = 100;
|
Chris@441
|
652 if (feature.values.size() > index) {
|
Chris@441
|
653 velocity = feature.values[index++];
|
Chris@441
|
654 }
|
Chris@441
|
655 if (velocity < 0) velocity = 127;
|
Chris@441
|
656 if (velocity > 127) velocity = 127;
|
Chris@320
|
657
|
Chris@441
|
658 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@441
|
659 if (!model) return;
|
Chris@441
|
660 model->addPoint(NoteModel::Point(frame, value, // value is pitch
|
Chris@441
|
661 lrintf(duration),
|
Chris@441
|
662 velocity / 127.f,
|
Chris@441
|
663 feature.label.c_str()));
|
Chris@441
|
664 } else {
|
Chris@441
|
665 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@441
|
666 if (model) {
|
Chris@441
|
667 model->addPoint(RegionModel::Point(frame, value,
|
Chris@441
|
668 lrintf(duration),
|
Chris@441
|
669 feature.label.c_str()));
|
Chris@441
|
670 } else return;
|
Chris@441
|
671 }
|
Chris@320
|
672
|
Chris@441
|
673 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
674
|
Chris@320
|
675 DenseThreeDimensionalModel::Column values = feature.values;
|
Chris@320
|
676
|
Chris@320
|
677 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
678 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
679 if (!model) return;
|
Chris@320
|
680
|
Chris@320
|
681 model->setColumn(frame / model->getResolution(), values);
|
Chris@441
|
682
|
Chris@441
|
683 } else {
|
Chris@441
|
684 std::cerr << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << std::endl;
|
Chris@320
|
685 }
|
Chris@320
|
686 }
|
Chris@320
|
687
|
Chris@320
|
688 void
|
Chris@331
|
689 FeatureExtractionModelTransformer::setCompletion(int completion)
|
Chris@320
|
690 {
|
Chris@320
|
691 int binCount = 1;
|
Chris@320
|
692 if (m_descriptor->hasFixedBinCount) {
|
Chris@320
|
693 binCount = m_descriptor->binCount;
|
Chris@320
|
694 }
|
Chris@320
|
695
|
Chris@331
|
696 // std::cerr << "FeatureExtractionModelTransformer::setCompletion("
|
Chris@320
|
697 // << completion << ")" << std::endl;
|
Chris@320
|
698
|
Chris@441
|
699 if (isOutput<SparseOneDimensionalModel>()) {
|
Chris@320
|
700
|
Chris@350
|
701 SparseOneDimensionalModel *model =
|
Chris@350
|
702 getConformingOutput<SparseOneDimensionalModel>();
|
Chris@320
|
703 if (!model) return;
|
Chris@441
|
704 model->setCompletion(completion, true);
|
Chris@320
|
705
|
Chris@441
|
706 } else if (isOutput<SparseTimeValueModel>()) {
|
Chris@320
|
707
|
Chris@350
|
708 SparseTimeValueModel *model =
|
Chris@350
|
709 getConformingOutput<SparseTimeValueModel>();
|
Chris@320
|
710 if (!model) return;
|
Chris@441
|
711 model->setCompletion(completion, true);
|
Chris@320
|
712
|
Chris@441
|
713 } else if (isOutput<NoteModel>()) {
|
Chris@320
|
714
|
Chris@441
|
715 NoteModel *model = getConformingOutput<NoteModel>();
|
Chris@320
|
716 if (!model) return;
|
Chris@441
|
717 model->setCompletion(completion, true);
|
Chris@320
|
718
|
Chris@441
|
719 } else if (isOutput<RegionModel>()) {
|
Chris@441
|
720
|
Chris@441
|
721 RegionModel *model = getConformingOutput<RegionModel>();
|
Chris@441
|
722 if (!model) return;
|
Chris@441
|
723 model->setCompletion(completion, true);
|
Chris@441
|
724
|
Chris@441
|
725 } else if (isOutput<EditableDenseThreeDimensionalModel>()) {
|
Chris@320
|
726
|
Chris@320
|
727 EditableDenseThreeDimensionalModel *model =
|
Chris@350
|
728 getConformingOutput<EditableDenseThreeDimensionalModel>();
|
Chris@320
|
729 if (!model) return;
|
Chris@350
|
730 model->setCompletion(completion, true); //!!!m_context.updates);
|
Chris@320
|
731 }
|
Chris@320
|
732 }
|
Chris@320
|
733
|