annotate transform/FeatureExtractionPluginTransform.cpp @ 0:da6937383da8

initial import
author Chris Cannam
date Tue, 10 Jan 2006 16:33:16 +0000
parents
children d86891498eef
rev   line source
Chris@0 1
Chris@0 2 /* -*- c-basic-offset: 4 -*- vi:set ts=8 sts=4 sw=4: */
Chris@0 3
Chris@0 4 /*
Chris@0 5 A waveform viewer and audio annotation editor.
Chris@0 6 Chris Cannam, Queen Mary University of London, 2005
Chris@0 7
Chris@0 8 This is experimental software. Not for distribution.
Chris@0 9 */
Chris@0 10
Chris@0 11 #include "FeatureExtractionPluginTransform.h"
Chris@0 12
Chris@0 13 #include "plugin/FeatureExtractionPluginFactory.h"
Chris@0 14 #include "plugin/FeatureExtractionPlugin.h"
Chris@0 15
Chris@0 16 #include "base/Model.h"
Chris@0 17 #include "model/SparseOneDimensionalModel.h"
Chris@0 18 #include "model/SparseTimeValueModel.h"
Chris@0 19 #include "model/DenseThreeDimensionalModel.h"
Chris@0 20 #include "model/DenseTimeValueModel.h"
Chris@0 21
Chris@0 22 #include <iostream>
Chris@0 23
Chris@0 24 FeatureExtractionPluginTransform::FeatureExtractionPluginTransform(Model *inputModel,
Chris@0 25 QString pluginId,
Chris@0 26 QString outputName) :
Chris@0 27 Transform(inputModel),
Chris@0 28 m_plugin(0),
Chris@0 29 m_descriptor(0),
Chris@0 30 m_outputFeatureNo(0)
Chris@0 31 {
Chris@0 32 std::cerr << "FeatureExtractionPluginTransform::FeatureExtractionPluginTransform: plugin " << pluginId.toStdString() << ", outputName " << outputName.toStdString() << std::endl;
Chris@0 33
Chris@0 34 FeatureExtractionPluginFactory *factory =
Chris@0 35 FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@0 36
Chris@0 37 if (!factory) {
Chris@0 38 std::cerr << "FeatureExtractionPluginTransform: No factory available for plugin id \""
Chris@0 39 << pluginId.toStdString() << "\"" << std::endl;
Chris@0 40 return;
Chris@0 41 }
Chris@0 42
Chris@0 43 m_plugin = factory->instantiatePlugin(pluginId, m_input->getSampleRate());
Chris@0 44
Chris@0 45 if (!m_plugin) {
Chris@0 46 std::cerr << "FeatureExtractionPluginTransform: Failed to instantiate plugin \""
Chris@0 47 << pluginId.toStdString() << "\"" << std::endl;
Chris@0 48 return;
Chris@0 49 }
Chris@0 50
Chris@0 51 FeatureExtractionPlugin::OutputList outputs =
Chris@0 52 m_plugin->getOutputDescriptors();
Chris@0 53
Chris@0 54 if (outputs.empty()) {
Chris@0 55 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
Chris@0 56 << pluginId.toStdString() << "\" has no outputs" << std::endl;
Chris@0 57 return;
Chris@0 58 }
Chris@0 59
Chris@0 60 for (size_t i = 0; i < outputs.size(); ++i) {
Chris@0 61 if (outputName == "" || outputs[i].name == outputName.toStdString()) {
Chris@0 62 m_outputFeatureNo = i;
Chris@0 63 m_descriptor = new FeatureExtractionPlugin::OutputDescriptor
Chris@0 64 (outputs[i]);
Chris@0 65 break;
Chris@0 66 }
Chris@0 67 }
Chris@0 68
Chris@0 69 if (!m_descriptor) {
Chris@0 70 std::cerr << "FeatureExtractionPluginTransform: Plugin \""
Chris@0 71 << pluginId.toStdString() << "\" has no output named \""
Chris@0 72 << outputName.toStdString() << "\"" << std::endl;
Chris@0 73 return;
Chris@0 74 }
Chris@0 75
Chris@0 76 std::cerr << "FeatureExtractionPluginTransform: output sample type "
Chris@0 77 << m_descriptor->sampleType << std::endl;
Chris@0 78
Chris@0 79 int valueCount = 1;
Chris@0 80 float minValue = 0.0, maxValue = 0.0;
Chris@0 81
Chris@0 82 if (m_descriptor->hasFixedValueCount) {
Chris@0 83 valueCount = m_descriptor->valueCount;
Chris@0 84 }
Chris@0 85
Chris@0 86 if (valueCount > 0 && m_descriptor->hasKnownExtents) {
Chris@0 87 minValue = m_descriptor->minValue;
Chris@0 88 maxValue = m_descriptor->maxValue;
Chris@0 89 }
Chris@0 90
Chris@0 91 size_t modelRate = m_input->getSampleRate();
Chris@0 92 size_t modelResolution = 1;
Chris@0 93
Chris@0 94 switch (m_descriptor->sampleType) {
Chris@0 95
Chris@0 96 case FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate:
Chris@0 97 if (m_descriptor->sampleRate != 0.0) {
Chris@0 98 modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@0 99 }
Chris@0 100 break;
Chris@0 101
Chris@0 102 case FeatureExtractionPlugin::OutputDescriptor::OneSamplePerStep:
Chris@0 103 modelResolution = m_plugin->getPreferredStepSize();
Chris@0 104 break;
Chris@0 105
Chris@0 106 case FeatureExtractionPlugin::OutputDescriptor::FixedSampleRate:
Chris@0 107 modelRate = m_descriptor->sampleRate;
Chris@0 108 break;
Chris@0 109 }
Chris@0 110
Chris@0 111 if (valueCount == 0) {
Chris@0 112
Chris@0 113 m_output = new SparseOneDimensionalModel(modelRate, modelResolution);
Chris@0 114
Chris@0 115 } else if (valueCount == 1 ||
Chris@0 116
Chris@0 117 // We don't have a sparse 3D model
Chris@0 118 m_descriptor->sampleType ==
Chris@0 119 FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0 120
Chris@0 121 m_output = new SparseTimeValueModel(modelRate, modelResolution,
Chris@0 122 minValue, maxValue, false);
Chris@0 123
Chris@0 124 } else {
Chris@0 125
Chris@0 126 m_output = new DenseThreeDimensionalModel(modelRate, modelResolution,
Chris@0 127 valueCount);
Chris@0 128 }
Chris@0 129 }
Chris@0 130
Chris@0 131 FeatureExtractionPluginTransform::~FeatureExtractionPluginTransform()
Chris@0 132 {
Chris@0 133 delete m_plugin;
Chris@0 134 delete m_descriptor;
Chris@0 135 }
Chris@0 136
Chris@0 137 DenseTimeValueModel *
Chris@0 138 FeatureExtractionPluginTransform::getInput()
Chris@0 139 {
Chris@0 140 DenseTimeValueModel *dtvm =
Chris@0 141 dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@0 142 if (!dtvm) {
Chris@0 143 std::cerr << "FeatureExtractionPluginTransform::getInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
Chris@0 144 }
Chris@0 145 return dtvm;
Chris@0 146 }
Chris@0 147
Chris@0 148 void
Chris@0 149 FeatureExtractionPluginTransform::run()
Chris@0 150 {
Chris@0 151 DenseTimeValueModel *input = getInput();
Chris@0 152 if (!input) return;
Chris@0 153
Chris@0 154 if (!m_output) return;
Chris@0 155
Chris@0 156 size_t channelCount = input->getChannelCount();
Chris@0 157 if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@0 158 channelCount = 1;
Chris@0 159 }
Chris@0 160 if (m_plugin->getMinChannelCount() > channelCount) {
Chris@0 161 std::cerr << "FeatureExtractionPluginTransform::run: "
Chris@0 162 << "Can't provide enough channels to plugin (plugin min "
Chris@0 163 << m_plugin->getMinChannelCount() << ", max "
Chris@0 164 << m_plugin->getMaxChannelCount() << ", input model has "
Chris@0 165 << input->getChannelCount() << ")" << std::endl;
Chris@0 166 return;
Chris@0 167 }
Chris@0 168
Chris@0 169 size_t sampleRate = m_input->getSampleRate();
Chris@0 170
Chris@0 171 size_t stepSize = m_plugin->getPreferredStepSize();
Chris@0 172 size_t blockSize = m_plugin->getPreferredBlockSize();
Chris@0 173
Chris@0 174 m_plugin->initialise(channelCount, stepSize, blockSize);
Chris@0 175
Chris@0 176 float **buffers = new float*[channelCount];
Chris@0 177 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@0 178 buffers[ch] = new float[blockSize];
Chris@0 179 }
Chris@0 180
Chris@0 181 size_t startFrame = m_input->getStartFrame();
Chris@0 182 size_t endFrame = m_input->getEndFrame();
Chris@0 183 size_t blockFrame = startFrame;
Chris@0 184
Chris@0 185 size_t prevCompletion = 0;
Chris@0 186
Chris@0 187 while (blockFrame < endFrame) {
Chris@0 188
Chris@0 189 // std::cerr << "FeatureExtractionPluginTransform::run: blockFrame "
Chris@0 190 // << blockFrame << std::endl;
Chris@0 191
Chris@0 192 size_t completion =
Chris@0 193 (((blockFrame - startFrame) / stepSize) * 99) /
Chris@0 194 ( (endFrame - startFrame) / stepSize);
Chris@0 195
Chris@0 196 // channelCount is either m_input->channelCount or 1
Chris@0 197
Chris@0 198 size_t got = 0;
Chris@0 199
Chris@0 200 if (channelCount == 1) {
Chris@0 201 got = input->getValues
Chris@0 202 (-1, blockFrame, blockFrame + blockSize, buffers[0]);
Chris@0 203 while (got < blockSize) {
Chris@0 204 buffers[0][got++] = 0.0;
Chris@0 205 }
Chris@0 206 } else {
Chris@0 207 for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@0 208 got = input->getValues
Chris@0 209 (ch, blockFrame, blockFrame + blockSize, buffers[ch]);
Chris@0 210 while (got < blockSize) {
Chris@0 211 buffers[ch][got++] = 0.0;
Chris@0 212 }
Chris@0 213 }
Chris@0 214 }
Chris@0 215
Chris@0 216 FeatureExtractionPlugin::FeatureSet features = m_plugin->process
Chris@0 217 (buffers, RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@0 218
Chris@0 219 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@0 220 FeatureExtractionPlugin::Feature feature =
Chris@0 221 features[m_outputFeatureNo][fi];
Chris@0 222 addFeature(blockFrame, feature);
Chris@0 223 }
Chris@0 224
Chris@0 225 if (blockFrame == startFrame || completion > prevCompletion) {
Chris@0 226 setCompletion(completion);
Chris@0 227 prevCompletion = completion;
Chris@0 228 }
Chris@0 229
Chris@0 230 blockFrame += stepSize;
Chris@0 231 }
Chris@0 232
Chris@0 233 FeatureExtractionPlugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@0 234
Chris@0 235 for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@0 236 FeatureExtractionPlugin::Feature feature =
Chris@0 237 features[m_outputFeatureNo][fi];
Chris@0 238 addFeature(blockFrame, feature);
Chris@0 239 }
Chris@0 240
Chris@0 241 setCompletion(100);
Chris@0 242 }
Chris@0 243
Chris@0 244
Chris@0 245 void
Chris@0 246 FeatureExtractionPluginTransform::addFeature(size_t blockFrame,
Chris@0 247 const FeatureExtractionPlugin::Feature &feature)
Chris@0 248 {
Chris@0 249 size_t inputRate = m_input->getSampleRate();
Chris@0 250
Chris@0 251 // std::cerr << "FeatureExtractionPluginTransform::addFeature("
Chris@0 252 // << blockFrame << ")" << std::endl;
Chris@0 253
Chris@0 254 int valueCount = 1;
Chris@0 255 if (m_descriptor->hasFixedValueCount) {
Chris@0 256 valueCount = m_descriptor->valueCount;
Chris@0 257 }
Chris@0 258
Chris@0 259 size_t frame = blockFrame;
Chris@0 260
Chris@0 261 if (m_descriptor->sampleType ==
Chris@0 262 FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0 263
Chris@0 264 if (!feature.hasTimestamp) {
Chris@0 265 std::cerr
Chris@0 266 << "WARNING: FeatureExtractionPluginTransform::addFeature: "
Chris@0 267 << "Feature has variable sample rate but no timestamp!"
Chris@0 268 << std::endl;
Chris@0 269 return;
Chris@0 270 } else {
Chris@0 271 frame = RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@0 272 }
Chris@0 273
Chris@0 274 } else if (m_descriptor->sampleType ==
Chris@0 275 FeatureExtractionPlugin::OutputDescriptor::FixedSampleRate) {
Chris@0 276
Chris@0 277 if (feature.hasTimestamp) {
Chris@0 278 //!!! warning: sampleRate may be non-integral
Chris@0 279 frame = RealTime::realTime2Frame(feature.timestamp,
Chris@0 280 m_descriptor->sampleRate);
Chris@0 281 } else {
Chris@0 282 frame = m_output->getEndFrame() + 1;
Chris@0 283 }
Chris@0 284 }
Chris@0 285
Chris@0 286 if (valueCount == 0) {
Chris@0 287
Chris@0 288 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
Chris@0 289 if (!model) return;
Chris@0 290 model->addPoint(SparseOneDimensionalModel::Point(frame, feature.label.c_str()));
Chris@0 291
Chris@0 292 } else if (valueCount == 1 ||
Chris@0 293 m_descriptor->sampleType ==
Chris@0 294 FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0 295
Chris@0 296 float value = 0.0;
Chris@0 297 if (feature.values.size() > 0) value = feature.values[0];
Chris@0 298
Chris@0 299 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
Chris@0 300 if (!model) return;
Chris@0 301 model->addPoint(SparseTimeValueModel::Point(frame, value, feature.label.c_str()));
Chris@0 302
Chris@0 303 } else {
Chris@0 304
Chris@0 305 DenseThreeDimensionalModel::BinValueSet values = feature.values;
Chris@0 306
Chris@0 307 DenseThreeDimensionalModel *model = getOutput<DenseThreeDimensionalModel>();
Chris@0 308 if (!model) return;
Chris@0 309
Chris@0 310 model->setBinValues(frame, values);
Chris@0 311 }
Chris@0 312 }
Chris@0 313
Chris@0 314 void
Chris@0 315 FeatureExtractionPluginTransform::setCompletion(int completion)
Chris@0 316 {
Chris@0 317 int valueCount = 1;
Chris@0 318 if (m_descriptor->hasFixedValueCount) {
Chris@0 319 valueCount = m_descriptor->valueCount;
Chris@0 320 }
Chris@0 321
Chris@0 322 if (valueCount == 0) {
Chris@0 323
Chris@0 324 SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
Chris@0 325 if (!model) return;
Chris@0 326 model->setCompletion(completion);
Chris@0 327
Chris@0 328 } else if (valueCount == 1 ||
Chris@0 329 m_descriptor->sampleType ==
Chris@0 330 FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0 331
Chris@0 332 SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
Chris@0 333 if (!model) return;
Chris@0 334 model->setCompletion(completion);
Chris@0 335
Chris@0 336 } else {
Chris@0 337
Chris@0 338 //!!! Can't actually do this with the 3D model (yet?)
Chris@0 339 }
Chris@0 340 }
Chris@0 341