comparison transform/FeatureExtractionModelTransformer.cpp @ 1739:565575463752 by-id

Some work on models and transformers
author Chris Cannam
date Wed, 26 Jun 2019 14:59:09 +0100
parents 5b7b01da430a
children fe3f7f8df3a3
comparison
equal deleted inserted replaced
1738:4abc0f08adf9 1739:565575463752
81 // All transforms must use the same plugin, parameters, and 81 // All transforms must use the same plugin, parameters, and
82 // inputs: they can differ only in choice of plugin output. So we 82 // inputs: they can differ only in choice of plugin output. So we
83 // initialise based purely on the first transform in the list (but 83 // initialise based purely on the first transform in the list (but
84 // first check that they are actually similar as promised) 84 // first check that they are actually similar as promised)
85 85
86 for (int j = 1; j < (int)m_transforms.size(); ++j) { 86 for (int j = 1; in_range_for(m_transforms, j); ++j) {
87 if (!areTransformsSimilar(m_transforms[0], m_transforms[j])) { 87 if (!areTransformsSimilar(m_transforms[0], m_transforms[j])) {
88 m_message = tr("Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output"); 88 m_message = tr("Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output");
89 SVCERR << m_message << endl; 89 SVCERR << m_message << endl;
90 return false; 90 return false;
91 } 91 }
102 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId); 102 m_message = tr("No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
103 SVCERR << m_message << endl; 103 SVCERR << m_message << endl;
104 return false; 104 return false;
105 } 105 }
106 106
107 DenseTimeValueModel *input = getConformingInput(); 107 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
108 if (!input) { 108 if (!input) {
109 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId); 109 m_message = tr("Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
110 SVCERR << m_message << endl; 110 SVCERR << m_message << endl;
111 return false; 111 return false;
112 } 112 }
156 if (step != preferredStep || block != preferredBlock) { 156 if (step != preferredStep || block != preferredBlock) {
157 157
158 SVDEBUG << "Initialisation failed, trying again with preferred step = " 158 SVDEBUG << "Initialisation failed, trying again with preferred step = "
159 << preferredStep << ", block = " << preferredBlock << endl; 159 << preferredStep << ", block = " << preferredBlock << endl;
160 160
161 if (!m_plugin->initialise(channelCount, preferredStep, preferredBlock)) { 161 if (!m_plugin->initialise(channelCount,
162 preferredStep,
163 preferredBlock)) {
162 164
163 SVDEBUG << "Initialisation failed again" << endl; 165 SVDEBUG << "Initialisation failed again" << endl;
164 166
165 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId); 167 m_message = tr("Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
166 SVCERR << m_message << endl; 168 SVCERR << m_message << endl;
219 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId); 221 m_message = tr("Plugin \"%1\" has no outputs").arg(pluginId);
220 SVCERR << m_message << endl; 222 SVCERR << m_message << endl;
221 return false; 223 return false;
222 } 224 }
223 225
224 for (int j = 0; j < (int)m_transforms.size(); ++j) { 226 for (int j = 0; in_range_for(m_transforms, j); ++j) {
225 227
226 for (int i = 0; i < (int)outputs.size(); ++i) { 228 for (int i = 0; in_range_for(outputs, i); ++i) {
227 // SVDEBUG << "comparing output " << i << " name \"" << outputs[i].identifier << "\" with expected \"" << m_transform.getOutput() << "\"" << endl; 229
228 if (m_transforms[j].getOutput() == "" || 230 if (m_transforms[j].getOutput() == "" ||
229 outputs[i].identifier == m_transforms[j].getOutput().toStdString()) { 231 outputs[i].identifier ==
232 m_transforms[j].getOutput().toStdString()) {
233
230 m_outputNos.push_back(i); 234 m_outputNos.push_back(i);
231 m_descriptors.push_back(new Vamp::Plugin::OutputDescriptor(outputs[i])); 235 m_descriptors.push_back(outputs[i]);
232 m_fixedRateFeatureNos.push_back(-1); // we increment before use 236 m_fixedRateFeatureNos.push_back(-1); // we increment before use
233 break; 237 break;
234 } 238 }
235 } 239 }
236 240
237 if ((int)m_descriptors.size() <= j) { 241 if (!in_range_for(m_descriptors, j)) {
238 m_message = tr("Plugin \"%1\" has no output named \"%2\"") 242 m_message = tr("Plugin \"%1\" has no output named \"%2\"")
239 .arg(pluginId) 243 .arg(pluginId)
240 .arg(m_transforms[j].getOutput()); 244 .arg(m_transforms[j].getOutput());
241 SVCERR << m_message << endl; 245 SVCERR << m_message << endl;
242 return false; 246 return false;
243 } 247 }
244 } 248 }
245 249
246 for (int j = 0; j < (int)m_transforms.size(); ++j) { 250 for (int j = 0; in_range_for(m_transforms, j); ++j) {
247 createOutputModels(j); 251 createOutputModels(j);
248 } 252 }
249 253
250 m_outputMutex.lock(); 254 m_outputMutex.lock();
251 m_haveOutputs = true; 255 m_haveOutputs = true;
269 // accidentally done so, so just in case: 273 // accidentally done so, so just in case:
270 SVCERR << "FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl; 274 SVCERR << "FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl;
271 m_message = e.what(); 275 m_message = e.what();
272 } 276 }
273 m_plugin = nullptr; 277 m_plugin = nullptr;
274 278
275 for (int j = 0; j < (int)m_descriptors.size(); ++j) { 279 m_descriptors.clear();
276 delete m_descriptors[j];
277 }
278 } 280 }
279 281
280 void 282 void
281 FeatureExtractionModelTransformer::createOutputModels(int n) 283 FeatureExtractionModelTransformer::createOutputModels(int n)
282 { 284 {
283 DenseTimeValueModel *input = getConformingInput(); 285 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
286 if (!input) return;
284 287
285 PluginRDFDescription description(m_transforms[n].getPluginIdentifier()); 288 PluginRDFDescription description(m_transforms[n].getPluginIdentifier());
286 QString outputId = m_transforms[n].getOutput(); 289 QString outputId = m_transforms[n].getOutput();
287 290
288 int binCount = 1; 291 int binCount = 1;
289 float minValue = 0.0, maxValue = 0.0; 292 float minValue = 0.0, maxValue = 0.0;
290 bool haveExtents = false; 293 bool haveExtents = false;
291 bool haveBinCount = m_descriptors[n]->hasFixedBinCount; 294 bool haveBinCount = m_descriptors[n].hasFixedBinCount;
292 295
293 if (haveBinCount) { 296 if (haveBinCount) {
294 binCount = (int)m_descriptors[n]->binCount; 297 binCount = (int)m_descriptors[n].binCount;
295 } 298 }
296 299
297 m_needAdditionalModels[n] = false; 300 m_needAdditionalModels[n] = false;
298 301
299 // cerr << "FeatureExtractionModelTransformer: output bin count " 302 if (binCount > 0 && m_descriptors[n].hasKnownExtents) {
300 // << binCount << endl; 303 minValue = m_descriptors[n].minValue;
301 304 maxValue = m_descriptors[n].maxValue;
302 if (binCount > 0 && m_descriptors[n]->hasKnownExtents) {
303 minValue = m_descriptors[n]->minValue;
304 maxValue = m_descriptors[n]->maxValue;
305 haveExtents = true; 305 haveExtents = true;
306 } 306 }
307 307
308 sv_samplerate_t modelRate = input->getSampleRate(); 308 sv_samplerate_t modelRate = input->getSampleRate();
309 sv_samplerate_t outputRate = modelRate; 309 sv_samplerate_t outputRate = modelRate;
310 int modelResolution = 1; 310 int modelResolution = 1;
311 311
312 if (m_descriptors[n]->sampleType != 312 if (m_descriptors[n].sampleType !=
313 Vamp::Plugin::OutputDescriptor::OneSamplePerStep) { 313 Vamp::Plugin::OutputDescriptor::OneSamplePerStep) {
314 314
315 outputRate = m_descriptors[n]->sampleRate; 315 outputRate = m_descriptors[n].sampleRate;
316 316
317 //!!! SV doesn't actually support display of models that have 317 //!!! SV doesn't actually support display of models that have
318 //!!! different underlying rates together -- so we always set 318 //!!! different underlying rates together -- so we always set
319 //!!! the model rate to be the input model's rate, and adjust 319 //!!! the model rate to be the input model's rate, and adjust
320 //!!! the resolution appropriately. We can't properly display 320 //!!! the resolution appropriately. We can't properly display
326 << modelRate << ")" << endl; 326 << modelRate << ")" << endl;
327 outputRate = modelRate; 327 outputRate = modelRate;
328 } 328 }
329 } 329 }
330 330
331 switch (m_descriptors[n]->sampleType) { 331 switch (m_descriptors[n].sampleType) {
332 332
333 case Vamp::Plugin::OutputDescriptor::VariableSampleRate: 333 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
334 if (outputRate != 0.0) { 334 if (outputRate != 0.0) {
335 modelResolution = int(round(modelRate / outputRate)); 335 modelResolution = int(round(modelRate / outputRate));
336 } 336 }
340 modelResolution = m_transforms[n].getStepSize(); 340 modelResolution = m_transforms[n].getStepSize();
341 break; 341 break;
342 342
343 case Vamp::Plugin::OutputDescriptor::FixedSampleRate: 343 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
344 if (outputRate <= 0.0) { 344 if (outputRate <= 0.0) {
345 SVDEBUG << "WARNING: Fixed sample-rate plugin reports invalid sample rate " << m_descriptors[n]->sampleRate << "; defaulting to input rate of " << input->getSampleRate() << endl; 345 SVDEBUG << "WARNING: Fixed sample-rate plugin reports invalid sample rate " << m_descriptors[n].sampleRate << "; defaulting to input rate of " << input->getSampleRate() << endl;
346 modelResolution = 1; 346 modelResolution = 1;
347 } else { 347 } else {
348 modelResolution = int(round(modelRate / outputRate)); 348 modelResolution = int(round(modelRate / outputRate));
349 // cerr << "modelRate = " << modelRate << ", descriptor rate = " << outputRate << ", modelResolution = " << modelResolution << endl; 349 // cerr << "modelRate = " << modelRate << ", descriptor rate = " << outputRate << ", modelResolution = " << modelResolution << endl;
350 } 350 }
351 break; 351 break;
352 } 352 }
353 353
354 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2); 354 bool preDurationPlugin = (m_plugin->getVampApiVersion() < 2);
355 355
356 Model *out = nullptr; 356 std::shared_ptr<Model> out;
357 357
358 if (binCount == 0 && 358 if (binCount == 0 &&
359 (preDurationPlugin || !m_descriptors[n]->hasDuration)) { 359 (preDurationPlugin || !m_descriptors[n].hasDuration)) {
360 360
361 // Anything with no value and no duration is an instant 361 // Anything with no value and no duration is an instant
362 362
363 out = new SparseOneDimensionalModel(modelRate, modelResolution, false); 363 out = std::make_shared<SparseOneDimensionalModel>
364 (modelRate, modelResolution, false);
365
364 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId); 366 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
365 out->setRDFTypeURI(outputEventTypeURI); 367 out->setRDFTypeURI(outputEventTypeURI);
366 368
367 } else if ((preDurationPlugin && binCount > 1 && 369 } else if ((preDurationPlugin && binCount > 1 &&
368 (m_descriptors[n]->sampleType == 370 (m_descriptors[n].sampleType ==
369 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) || 371 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
370 (!preDurationPlugin && m_descriptors[n]->hasDuration)) { 372 (!preDurationPlugin && m_descriptors[n].hasDuration)) {
371 373
372 // For plugins using the old v1 API without explicit duration, 374 // For plugins using the old v1 API without explicit duration,
373 // we treat anything that has multiple bins (i.e. that has the 375 // we treat anything that has multiple bins (i.e. that has the
374 // potential to have value and duration) and a variable sample 376 // potential to have value and duration) and a variable sample
375 // rate as a note model, taking its values as pitch, duration 377 // rate as a note model, taking its values as pitch, duration
396 // duration) 398 // duration)
397 if (binCount > 1) isNoteModel = true; 399 if (binCount > 1) isNoteModel = true;
398 400
399 // Regions do not have units of Hz or MIDI things (a sweeping 401 // Regions do not have units of Hz or MIDI things (a sweeping
400 // assumption!) 402 // assumption!)
401 if (m_descriptors[n]->unit == "Hz" || 403 if (m_descriptors[n].unit == "Hz" ||
402 m_descriptors[n]->unit.find("MIDI") != std::string::npos || 404 m_descriptors[n].unit.find("MIDI") != std::string::npos ||
403 m_descriptors[n]->unit.find("midi") != std::string::npos) { 405 m_descriptors[n].unit.find("midi") != std::string::npos) {
404 isNoteModel = true; 406 isNoteModel = true;
405 } 407 }
406 408
407 // If we had a "sparse 3D model", we would have the additional 409 // If we had a "sparse 3D model", we would have the additional
408 // problem of determining whether to use that here (if bin 410 // problem of determining whether to use that here (if bin
418 (modelRate, modelResolution, minValue, maxValue, false); 420 (modelRate, modelResolution, minValue, maxValue, false);
419 } else { 421 } else {
420 model = new NoteModel 422 model = new NoteModel
421 (modelRate, modelResolution, false); 423 (modelRate, modelResolution, false);
422 } 424 }
423 model->setScaleUnits(m_descriptors[n]->unit.c_str()); 425 model->setScaleUnits(m_descriptors[n].unit.c_str());
424 out = model; 426 out.reset(model);
425 427
426 } else { 428 } else {
427 429
428 RegionModel *model; 430 RegionModel *model;
429 if (haveExtents) { 431 if (haveExtents) {
431 (modelRate, modelResolution, minValue, maxValue, false); 433 (modelRate, modelResolution, minValue, maxValue, false);
432 } else { 434 } else {
433 model = new RegionModel 435 model = new RegionModel
434 (modelRate, modelResolution, false); 436 (modelRate, modelResolution, false);
435 } 437 }
436 model->setScaleUnits(m_descriptors[n]->unit.c_str()); 438 model->setScaleUnits(m_descriptors[n].unit.c_str());
437 out = model; 439 out.reset(model);
438 } 440 }
439 441
440 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId); 442 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
441 out->setRDFTypeURI(outputEventTypeURI); 443 out->setRDFTypeURI(outputEventTypeURI);
442 444
443 } else if (binCount == 1 || 445 } else if (binCount == 1 ||
444 (m_descriptors[n]->sampleType == 446 (m_descriptors[n].sampleType ==
445 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) { 447 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
446 448
447 // Anything that is not a 1D, note, or interval model and that 449 // Anything that is not a 1D, note, or interval model and that
448 // has only one value per result must be a sparse time value 450 // has only one value per result must be a sparse time value
449 // model. 451 // model.
480 } 482 }
481 483
482 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors(); 484 Vamp::Plugin::OutputList outputs = m_plugin->getOutputDescriptors();
483 model->setScaleUnits(outputs[m_outputNos[n]].unit.c_str()); 485 model->setScaleUnits(outputs[m_outputNos[n]].unit.c_str());
484 486
485 out = model; 487 out.reset(model);
486 488
487 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId); 489 QString outputEventTypeURI = description.getOutputEventTypeURI(outputId);
488 out->setRDFTypeURI(outputEventTypeURI); 490 out->setRDFTypeURI(outputEventTypeURI);
489 491
490 } else { 492 } else {
497 new EditableDenseThreeDimensionalModel 499 new EditableDenseThreeDimensionalModel
498 (modelRate, modelResolution, binCount, 500 (modelRate, modelResolution, binCount,
499 EditableDenseThreeDimensionalModel::BasicMultirateCompression, 501 EditableDenseThreeDimensionalModel::BasicMultirateCompression,
500 false); 502 false);
501 503
502 if (!m_descriptors[n]->binNames.empty()) { 504 if (!m_descriptors[n].binNames.empty()) {
503 std::vector<QString> names; 505 std::vector<QString> names;
504 for (int i = 0; i < (int)m_descriptors[n]->binNames.size(); ++i) { 506 for (int i = 0; i < (int)m_descriptors[n].binNames.size(); ++i) {
505 names.push_back(m_descriptors[n]->binNames[i].c_str()); 507 names.push_back(m_descriptors[n].binNames[i].c_str());
506 } 508 }
507 model->setBinNames(names); 509 model->setBinNames(names);
508 } 510 }
509 511
510 out = model; 512 out.reset(model);
511 513
512 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId); 514 QString outputSignalTypeURI = description.getOutputSignalTypeURI(outputId);
513 out->setRDFTypeURI(outputSignalTypeURI); 515 out->setRDFTypeURI(outputSignalTypeURI);
514 } 516 }
515 517
516 if (out) { 518 if (out) {
517 out->setSourceModel(input); 519 out->setSourceModel(getInputModel());
518 m_outputs.push_back(out); 520 ModelById::add(out);
521 m_outputs.push_back(out->getId());
519 } 522 }
520 } 523 }
521 524
522 void 525 void
523 FeatureExtractionModelTransformer::awaitOutputModels() 526 FeatureExtractionModelTransformer::awaitOutputModels()
538 541
539 FeatureExtractionModelTransformer::Models 542 FeatureExtractionModelTransformer::Models
540 FeatureExtractionModelTransformer::getAdditionalOutputModels() 543 FeatureExtractionModelTransformer::getAdditionalOutputModels()
541 { 544 {
542 Models mm; 545 Models mm;
543 for (AdditionalModelMap::iterator i = m_additionalModels.begin(); 546 for (auto mp : m_additionalModels) {
544 i != m_additionalModels.end(); ++i) { 547 for (auto m: mp.second) {
545 for (std::map<int, SparseTimeValueModel *>::iterator j = 548 mm.push_back(m.second);
546 i->second.begin();
547 j != i->second.end(); ++j) {
548 SparseTimeValueModel *m = j->second;
549 if (m) mm.push_back(m);
550 } 549 }
551 } 550 }
552 return mm; 551 return mm;
553 } 552 }
554 553
555 bool 554 bool
556 FeatureExtractionModelTransformer::willHaveAdditionalOutputModels() 555 FeatureExtractionModelTransformer::willHaveAdditionalOutputModels()
557 { 556 {
558 for (std::map<int, bool>::const_iterator i = 557 for (auto p : m_needAdditionalModels) {
559 m_needAdditionalModels.begin(); 558 if (p.second) return true;
560 i != m_needAdditionalModels.end(); ++i) {
561 if (i->second) return true;
562 } 559 }
563 return false; 560 return false;
564 } 561 }
565 562
566 SparseTimeValueModel * 563 ModelId
567 FeatureExtractionModelTransformer::getAdditionalModel(int n, int binNo) 564 FeatureExtractionModelTransformer::getAdditionalModel(int n, int binNo)
568 { 565 {
569 // std::cerr << "getAdditionalModel(" << n << ", " << binNo << ")" << std::endl;
570
571 if (binNo == 0) { 566 if (binNo == 0) {
572 std::cerr << "Internal error: binNo == 0 in getAdditionalModel (should be using primary model)" << std::endl; 567 SVCERR << "Internal error: binNo == 0 in getAdditionalModel (should be using primary model, not calling getAdditionalModel)" << endl;
573 return nullptr; 568 return {};
574 } 569 }
575 570
576 if (!m_needAdditionalModels[n]) return nullptr; 571 if (!in_range_for(m_outputs, n)) {
577 if (!isOutput<SparseTimeValueModel>(n)) return nullptr; 572 SVCERR << "getAdditionalModel: Output " << n << " out of range" << endl;
578 if (m_additionalModels[n][binNo]) return m_additionalModels[n][binNo]; 573 return {};
579 574 }
580 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): creating" << std::endl; 575
581 576 if (!in_range_for(m_needAdditionalModels, n) ||
582 SparseTimeValueModel *baseModel = getConformingOutput<SparseTimeValueModel>(n); 577 !m_needAdditionalModels[n]) {
583 if (!baseModel) return nullptr; 578 return {};
584 579 }
585 std::cerr << "getAdditionalModel(" << n << ", " << binNo << "): (from " << baseModel << ")" << std::endl; 580
581 if (!m_additionalModels[n][binNo].isNone()) {
582 return m_additionalModels[n][binNo];
583 }
584
585 SVDEBUG << "getAdditionalModel(" << n << ", " << binNo
586 << "): creating" << endl;
587
588 auto baseModel = ModelById::getAs<SparseTimeValueModel>(m_outputs[n]);
589 if (!baseModel) {
590 SVCERR << "getAdditionalModel: Output model not conformable, or has vanished" << endl;
591 return {};
592 }
593
594 SVDEBUG << "getAdditionalModel(" << n << ", " << binNo
595 << "): (from " << baseModel << ")" << endl;
586 596
587 SparseTimeValueModel *additional = 597 SparseTimeValueModel *additional =
588 new SparseTimeValueModel(baseModel->getSampleRate(), 598 new SparseTimeValueModel(baseModel->getSampleRate(),
589 baseModel->getResolution(), 599 baseModel->getResolution(),
590 baseModel->getValueMinimum(), 600 baseModel->getValueMinimum(),
592 false); 602 false);
593 603
594 additional->setScaleUnits(baseModel->getScaleUnits()); 604 additional->setScaleUnits(baseModel->getScaleUnits());
595 additional->setRDFTypeURI(baseModel->getRDFTypeURI()); 605 additional->setRDFTypeURI(baseModel->getRDFTypeURI());
596 606
597 m_additionalModels[n][binNo] = additional; 607 ModelId additionalId = additional->getId();
598 return additional; 608 ModelById::add(std::shared_ptr<SparseTimeValueModel>(additional));
599 } 609 m_additionalModels[n][binNo] = additionalId;
600 610 return additionalId;
601 DenseTimeValueModel *
602 FeatureExtractionModelTransformer::getConformingInput()
603 {
604 // SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: input model is " << getInputModel() << endl;
605
606 DenseTimeValueModel *dtvm =
607 dynamic_cast<DenseTimeValueModel *>(getInputModel());
608 if (!dtvm) {
609 SVDEBUG << "FeatureExtractionModelTransformer::getConformingInput: WARNING: Input model is not conformable to DenseTimeValueModel" << endl;
610 }
611 return dtvm;
612 } 611 }
613 612
614 void 613 void
615 FeatureExtractionModelTransformer::run() 614 FeatureExtractionModelTransformer::run()
616 { 615 {
622 } catch (const std::exception &e) { 621 } catch (const std::exception &e) {
623 abandon(); 622 abandon();
624 m_message = e.what(); 623 m_message = e.what();
625 return; 624 return;
626 } 625 }
627
628 DenseTimeValueModel *input = getConformingInput();
629 if (!input) {
630 abandon();
631 return;
632 }
633 626
634 if (m_outputs.empty()) { 627 if (m_outputs.empty()) {
635 abandon(); 628 abandon();
636 return; 629 return;
637 } 630 }
638 631
639 Transform primaryTransform = m_transforms[0]; 632 Transform primaryTransform = m_transforms[0];
640 633
641 while (!input->isReady() && !m_abandoned) { 634 bool ready = false;
642 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl; 635 while (!ready && !m_abandoned) {
643 usleep(500000); 636 { // scope so as to release input shared_ptr before sleeping
644 } 637 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
645 SVDEBUG << "FeatureExtractionModelTransformer::run: Waited, ready = " 638 if (!input) {
646 << input->isReady() << ", m_abandoned = " << m_abandoned << endl; 639 abandon();
640 return;
641 }
642 ready = input->isReady();
643 }
644 if (!ready) {
645 SVDEBUG << "FeatureExtractionModelTransformer::run: Waiting for input model to be ready..." << endl;
646 usleep(500000);
647 }
648 }
647 if (m_abandoned) return; 649 if (m_abandoned) return;
650
651 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
652 if (!input) {
653 SVCERR << "FeatureExtractionModelTransformer::run: Input model not (no longer?) available, abandoning" << endl;
654 abandon();
655 return;
656 }
648 657
649 sv_samplerate_t sampleRate = input->getSampleRate(); 658 sv_samplerate_t sampleRate = input->getSampleRate();
650 659
651 int channelCount = input->getChannelCount(); 660 int channelCount = input->getChannelCount();
652 if ((int)m_plugin->getMaxChannelCount() < channelCount) { 661 if ((int)m_plugin->getMaxChannelCount() < channelCount) {
667 std::vector<FFTModel *> fftModels; 676 std::vector<FFTModel *> fftModels;
668 677
669 if (frequencyDomain) { 678 if (frequencyDomain) {
670 for (int ch = 0; ch < channelCount; ++ch) { 679 for (int ch = 0; ch < channelCount; ++ch) {
671 FFTModel *model = new FFTModel 680 FFTModel *model = new FFTModel
672 (getConformingInput(), 681 //!!! (input->getId(),
673 channelCount == 1 ? m_input.getChannel() : ch, 682 (nullptr,
674 primaryTransform.getWindowType(), 683 channelCount == 1 ? m_input.getChannel() : ch,
675 blockSize, 684 primaryTransform.getWindowType(),
676 stepSize, 685 blockSize,
677 blockSize); 686 stepSize,
687 blockSize);
678 if (!model->isOK() || model->getError() != "") { 688 if (!model->isOK() || model->getError() != "") {
679 QString err = model->getError(); 689 QString err = model->getError();
680 delete model; 690 delete model;
681 for (int j = 0; j < (int)m_outputNos.size(); ++j) { 691 for (int j = 0; in_range_for(m_outputNos, j); ++j) {
682 setCompletion(j, 100); 692 setCompletion(j, 100);
683 } 693 }
684 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either 694 //!!! need a better way to handle this -- previously we were using a QMessageBox but that isn't an appropriate thing to do here either
685 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer: error is: " + err); 695 throw AllocationFailed("Failed to create the FFT model for this feature extraction model transformer: error is: " + err);
686 } 696 }
687 fftModels.push_back(model); 697 fftModels.push_back(model);
688 cerr << "created model for channel " << ch << endl; 698 }
689 } 699 }
690 } 700
691 701 sv_frame_t startFrame = input->getStartFrame();
692 sv_frame_t startFrame = m_input.getModel()->getStartFrame(); 702 sv_frame_t endFrame = input->getEndFrame();
693 sv_frame_t endFrame = m_input.getModel()->getEndFrame();
694 703
695 RealTime contextStartRT = primaryTransform.getStartTime(); 704 RealTime contextStartRT = primaryTransform.getStartTime();
696 RealTime contextDurationRT = primaryTransform.getDuration(); 705 RealTime contextDurationRT = primaryTransform.getDuration();
697 706
698 sv_frame_t contextStart = 707 sv_frame_t contextStart =
714 723
715 sv_frame_t blockFrame = contextStart; 724 sv_frame_t blockFrame = contextStart;
716 725
717 long prevCompletion = 0; 726 long prevCompletion = 0;
718 727
719 for (int j = 0; j < (int)m_outputNos.size(); ++j) { 728 for (int j = 0; in_range_for(m_outputNos, j); ++j) {
720 setCompletion(j, 0); 729 setCompletion(j, 0);
721 } 730 }
722 731
723 float *reals = nullptr; 732 float *reals = nullptr;
724 float *imaginaries = nullptr; 733 float *imaginaries = nullptr;
732 try { 741 try {
733 while (!m_abandoned) { 742 while (!m_abandoned) {
734 743
735 if (frequencyDomain) { 744 if (frequencyDomain) {
736 if (blockFrame - int(blockSize)/2 > 745 if (blockFrame - int(blockSize)/2 >
737 contextStart + contextDuration) break; 746 contextStart + contextDuration) {
747 break;
748 }
738 } else { 749 } else {
739 if (blockFrame >= 750 if (blockFrame >=
740 contextStart + contextDuration) break; 751 contextStart + contextDuration) {
752 break;
753 }
741 } 754 }
742 755
743 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 756 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
744 SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame " 757 SVDEBUG << "FeatureExtractionModelTransformer::run: blockFrame "
745 << blockFrame << ", endFrame " << endFrame << ", blockSize " 758 << blockFrame << ", endFrame " << endFrame << ", blockSize "
748 761
749 int completion = int 762 int completion = int
750 ((((blockFrame - contextStart) / stepSize) * 99) / 763 ((((blockFrame - contextStart) / stepSize) * 99) /
751 (contextDuration / stepSize + 1)); 764 (contextDuration / stepSize + 1));
752 765
753 // channelCount is either m_input.getModel()->channelCount or 1 766 // channelCount is either input->channelCount or 1
754 767
755 if (frequencyDomain) { 768 if (frequencyDomain) {
756 for (int ch = 0; ch < channelCount; ++ch) { 769 for (int ch = 0; ch < channelCount; ++ch) {
757 int column = int((blockFrame - startFrame) / stepSize); 770 int column = int((blockFrame - startFrame) / stepSize);
758 if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) { 771 if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) {
861 size -= offset; 874 size -= offset;
862 if (size <= 0) return; 875 if (size <= 0) return;
863 startFrame = 0; 876 startFrame = 0;
864 } 877 }
865 878
866 DenseTimeValueModel *input = getConformingInput(); 879 auto input = ModelById::getAs<DenseTimeValueModel>(getInputModel());
867 if (!input) return; 880 if (!input) {
881 return;
882 }
868 883
869 sv_frame_t got = 0; 884 sv_frame_t got = 0;
870 885
871 if (channelCount == 1) { 886 if (channelCount == 1) {
872 887
905 void 920 void
906 FeatureExtractionModelTransformer::addFeature(int n, 921 FeatureExtractionModelTransformer::addFeature(int n,
907 sv_frame_t blockFrame, 922 sv_frame_t blockFrame,
908 const Vamp::Plugin::Feature &feature) 923 const Vamp::Plugin::Feature &feature)
909 { 924 {
910 sv_samplerate_t inputRate = m_input.getModel()->getSampleRate(); 925 auto input = ModelById::get(getInputModel());
926 if (!input) return;
927
928 sv_samplerate_t inputRate = input->getSampleRate();
911 929
912 // cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = " 930 // cerr << "FeatureExtractionModelTransformer::addFeature: blockFrame = "
913 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp 931 // << blockFrame << ", hasTimestamp = " << feature.hasTimestamp
914 // << ", timestamp = " << feature.timestamp << ", hasDuration = " 932 // << ", timestamp = " << feature.timestamp << ", hasDuration = "
915 // << feature.hasDuration << ", duration = " << feature.duration 933 // << feature.hasDuration << ", duration = " << feature.duration
916 // << endl; 934 // << endl;
917 935
918 sv_frame_t frame = blockFrame; 936 sv_frame_t frame = blockFrame;
919 937
920 if (m_descriptors[n]->sampleType == 938 if (m_descriptors[n].sampleType ==
921 Vamp::Plugin::OutputDescriptor::VariableSampleRate) { 939 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
922 940
923 if (!feature.hasTimestamp) { 941 if (!feature.hasTimestamp) {
924 SVDEBUG 942 SVDEBUG
925 << "WARNING: FeatureExtractionModelTransformer::addFeature: " 943 << "WARNING: FeatureExtractionModelTransformer::addFeature: "
931 } 949 }
932 950
933 // cerr << "variable sample rate: timestamp = " << feature.timestamp 951 // cerr << "variable sample rate: timestamp = " << feature.timestamp
934 // << " at input rate " << inputRate << " -> " << frame << endl; 952 // << " at input rate " << inputRate << " -> " << frame << endl;
935 953
936 } else if (m_descriptors[n]->sampleType == 954 } else if (m_descriptors[n].sampleType ==
937 Vamp::Plugin::OutputDescriptor::FixedSampleRate) { 955 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
938 956
939 sv_samplerate_t rate = m_descriptors[n]->sampleRate; 957 sv_samplerate_t rate = m_descriptors[n].sampleRate;
940 if (rate <= 0.0) { 958 if (rate <= 0.0) {
941 rate = inputRate; 959 rate = inputRate;
942 } 960 }
943 961
944 if (!feature.hasTimestamp) { 962 if (!feature.hasTimestamp) {
947 RealTime ts(feature.timestamp.sec, feature.timestamp.nsec); 965 RealTime ts(feature.timestamp.sec, feature.timestamp.nsec);
948 m_fixedRateFeatureNos[n] = (int)lrint(ts.toDouble() * rate); 966 m_fixedRateFeatureNos[n] = (int)lrint(ts.toDouble() * rate);
949 } 967 }
950 968
951 // cerr << "m_fixedRateFeatureNo = " << m_fixedRateFeatureNos[n] 969 // cerr << "m_fixedRateFeatureNo = " << m_fixedRateFeatureNos[n]
952 // << ", m_descriptor->sampleRate = " << m_descriptors[n]->sampleRate 970 // << ", m_descriptor->sampleRate = " << m_descriptors[n].sampleRate
953 // << ", inputRate = " << inputRate 971 // << ", inputRate = " << inputRate
954 // << " giving frame = "; 972 // << " giving frame = ";
955 frame = lrint((double(m_fixedRateFeatureNos[n]) / rate) * inputRate); 973 frame = lrint((double(m_fixedRateFeatureNos[n]) / rate) * inputRate);
956 // cerr << frame << endl; 974 // cerr << frame << endl;
957 } 975 }
969 // Rather than repeat the complicated tests from the constructor 987 // Rather than repeat the complicated tests from the constructor
970 // to determine what sort of model we must be adding the features 988 // to determine what sort of model we must be adding the features
971 // to, we instead test what sort of model the constructor decided 989 // to, we instead test what sort of model the constructor decided
972 // to create. 990 // to create.
973 991
974 if (isOutput<SparseOneDimensionalModel>(n)) { 992 ModelId outputId = m_outputs[n];
975 993 bool found = false;
976 SparseOneDimensionalModel *model = 994
977 getConformingOutput<SparseOneDimensionalModel>(n); 995 if (!found) {
978 if (!model) return; 996 auto model = ModelById::getAs<SparseOneDimensionalModel>(outputId);
979 997 if (model) {
980 model->add(Event(frame, feature.label.c_str())); 998 found = true;
999 model->add(Event(frame, feature.label.c_str()));
1000 }
1001 }
1002
1003 if (!found) {
1004 auto model = ModelById::getAs<SparseTimeValueModel>(outputId);
1005 if (model) {
1006 found = true;
1007
1008 for (int i = 0; in_range_for(feature.values, i); ++i) {
1009
1010 float value = feature.values[i];
1011
1012 QString label = feature.label.c_str();
1013 if (feature.values.size() > 1) {
1014 label = QString("[%1] %2").arg(i+1).arg(label);
1015 }
1016
1017 auto targetModel = model;
1018
1019 if (m_needAdditionalModels[n] && i > 0) {
1020 targetModel = ModelById::getAs<SparseTimeValueModel>
1021 (getAdditionalModel(n, i));
1022 if (!targetModel) targetModel = model;
1023 }
1024
1025 targetModel->add(Event(frame, value, label));
1026 }
1027 }
1028 }
1029
1030 if (!found) {
1031 if (ModelById::getAs<NoteModel>(outputId) ||
1032 ModelById::getAs<RegionModel>(outputId)) {
1033 found = true;
1034
1035 int index = 0;
1036
1037 float value = 0.0;
1038 if ((int)feature.values.size() > index) {
1039 value = feature.values[index++];
1040 }
1041
1042 sv_frame_t duration = 1;
1043 if (feature.hasDuration) {
1044 duration = RealTime::realTime2Frame(feature.duration, inputRate);
1045 } else {
1046 if (in_range_for(feature.values, index)) {
1047 duration = lrintf(feature.values[index++]);
1048 }
1049 }
1050
1051 auto noteModel = ModelById::getAs<NoteModel>(outputId);
1052 if (noteModel) {
1053
1054 float velocity = 100;
1055 if ((int)feature.values.size() > index) {
1056 velocity = feature.values[index++];
1057 }
1058 if (velocity < 0) velocity = 127;
1059 if (velocity > 127) velocity = 127;
1060
1061 noteModel->add(Event(frame, value, // value is pitch
1062 duration,
1063 velocity / 127.f,
1064 feature.label.c_str()));
1065 }
1066
1067 auto regionModel = ModelById::getAs<RegionModel>(outputId);
1068 if (regionModel) {
1069
1070 if (feature.hasDuration && !feature.values.empty()) {
1071
1072 for (int i = 0; in_range_for(feature.values, i); ++i) {
1073
1074 float value = feature.values[i];
1075
1076 QString label = feature.label.c_str();
1077 if (feature.values.size() > 1) {
1078 label = QString("[%1] %2").arg(i+1).arg(label);
1079 }
1080
1081 regionModel->add(Event(frame,
1082 value,
1083 duration,
1084 label));
1085 }
1086 } else {
1087
1088 regionModel->add(Event(frame,
1089 value,
1090 duration,
1091 feature.label.c_str()));
1092 }
1093 }
1094 }
1095 }
1096
1097 if (!found) {
1098 auto model = ModelById::getAs
1099 <EditableDenseThreeDimensionalModel>(outputId);
1100 if (model) {
1101 found = true;
981 1102
982 } else if (isOutput<SparseTimeValueModel>(n)) { 1103 DenseThreeDimensionalModel::Column values = feature.values;
983 1104
984 SparseTimeValueModel *model = 1105 if (!feature.hasTimestamp && m_fixedRateFeatureNos[n] >= 0) {
985 getConformingOutput<SparseTimeValueModel>(n); 1106 model->setColumn(m_fixedRateFeatureNos[n], values);
986 if (!model) return;
987
988 for (int i = 0; i < (int)feature.values.size(); ++i) {
989
990 float value = feature.values[i];
991
992 QString label = feature.label.c_str();
993 if (feature.values.size() > 1) {
994 label = QString("[%1] %2").arg(i+1).arg(label);
995 }
996
997 SparseTimeValueModel *targetModel = model;
998
999 if (m_needAdditionalModels[n] && i > 0) {
1000 targetModel = getAdditionalModel(n, i);
1001 if (!targetModel) targetModel = model;
1002 // std::cerr << "adding point to model " << targetModel
1003 // << " for output " << n << " bin " << i << std::endl;
1004 }
1005
1006 targetModel->add(Event(frame, value, label));
1007 }
1008
1009 } else if (isOutput<NoteModel>(n) || isOutput<RegionModel>(n)) {
1010
1011 int index = 0;
1012
1013 float value = 0.0;
1014 if ((int)feature.values.size() > index) {
1015 value = feature.values[index++];
1016 }
1017
1018 sv_frame_t duration = 1;
1019 if (feature.hasDuration) {
1020 duration = RealTime::realTime2Frame(feature.duration, inputRate);
1021 } else {
1022 if (in_range_for(feature.values, index)) {
1023 duration = lrintf(feature.values[index++]);
1024 }
1025 }
1026
1027 if (isOutput<NoteModel>(n)) {
1028
1029 float velocity = 100;
1030 if ((int)feature.values.size() > index) {
1031 velocity = feature.values[index++];
1032 }
1033 if (velocity < 0) velocity = 127;
1034 if (velocity > 127) velocity = 127;
1035
1036 NoteModel *model = getConformingOutput<NoteModel>(n);
1037 if (!model) return;
1038 model->add(Event(frame, value, // value is pitch
1039 duration,
1040 velocity / 127.f,
1041 feature.label.c_str()));
1042 } else {
1043
1044 RegionModel *model = getConformingOutput<RegionModel>(n);
1045 if (!model) return;
1046
1047 if (feature.hasDuration && !feature.values.empty()) {
1048
1049 for (int i = 0; i < (int)feature.values.size(); ++i) {
1050
1051 float value = feature.values[i];
1052
1053 QString label = feature.label.c_str();
1054 if (feature.values.size() > 1) {
1055 label = QString("[%1] %2").arg(i+1).arg(label);
1056 }
1057
1058 model->add(Event(frame,
1059 value,
1060 duration,
1061 label));
1062 }
1063 } else { 1107 } else {
1064 1108 model->setColumn(int(frame / model->getResolution()), values);
1065 model->add(Event(frame, 1109 }
1066 value, 1110 }
1067 duration, 1111 }
1068 feature.label.c_str())); 1112
1069 } 1113 if (!found) {
1070 }
1071
1072 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) {
1073
1074 DenseThreeDimensionalModel::Column values = feature.values;
1075
1076 EditableDenseThreeDimensionalModel *model =
1077 getConformingOutput<EditableDenseThreeDimensionalModel>(n);
1078 if (!model) return;
1079
1080 // cerr << "(note: model resolution = " << model->getResolution() << ")"
1081 // << endl;
1082
1083 if (!feature.hasTimestamp && m_fixedRateFeatureNos[n] >= 0) {
1084 model->setColumn(m_fixedRateFeatureNos[n], values);
1085 } else {
1086 model->setColumn(int(frame / model->getResolution()), values);
1087 }
1088
1089 } else {
1090 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl; 1114 SVDEBUG << "FeatureExtractionModelTransformer::addFeature: Unknown output model type!" << endl;
1091 } 1115 }
1092 } 1116 }
1093 1117
1094 void 1118 void
1097 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 1121 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN
1098 SVDEBUG << "FeatureExtractionModelTransformer::setCompletion(" 1122 SVDEBUG << "FeatureExtractionModelTransformer::setCompletion("
1099 << completion << ")" << endl; 1123 << completion << ")" << endl;
1100 #endif 1124 #endif
1101 1125
1102 if (isOutput<SparseOneDimensionalModel>(n)) { 1126 ModelId outputId = m_outputs[n];
1103 1127 bool found = false;
1104 SparseOneDimensionalModel *model = 1128
1105 getConformingOutput<SparseOneDimensionalModel>(n); 1129 if (!found) {
1106 if (!model) return; 1130 auto model = ModelById::getAs<SparseOneDimensionalModel>(outputId);
1107 if (model->isAbandoning()) abandon(); 1131 if (model) {
1108 model->setCompletion(completion, true); 1132 found = true;
1109 1133 model->setCompletion(completion, true);
1110 } else if (isOutput<SparseTimeValueModel>(n)) { 1134 }
1111 1135 }
1112 SparseTimeValueModel *model = 1136
1113 getConformingOutput<SparseTimeValueModel>(n); 1137 if (!found) {
1114 if (!model) return; 1138 auto model = ModelById::getAs<SparseTimeValueModel>(outputId);
1115 if (model->isAbandoning()) abandon(); 1139 if (model) {
1116 model->setCompletion(completion, true); 1140 found = true;
1117 1141 model->setCompletion(completion, true);
1118 } else if (isOutput<NoteModel>(n)) { 1142 }
1119 1143 }
1120 NoteModel *model = getConformingOutput<NoteModel>(n); 1144
1121 if (!model) return; 1145 if (!found) {
1122 if (model->isAbandoning()) abandon(); 1146 auto model = ModelById::getAs<NoteModel>(outputId);
1123 model->setCompletion(completion, true); 1147 if (model) {
1124 1148 found = true;
1125 } else if (isOutput<RegionModel>(n)) { 1149 model->setCompletion(completion, true);
1126 1150 }
1127 RegionModel *model = getConformingOutput<RegionModel>(n); 1151 }
1128 if (!model) return; 1152
1129 if (model->isAbandoning()) abandon(); 1153 if (!found) {
1130 model->setCompletion(completion, true); 1154 auto model = ModelById::getAs<RegionModel>(outputId);
1131 1155 if (model) {
1132 } else if (isOutput<EditableDenseThreeDimensionalModel>(n)) { 1156 found = true;
1133 1157 model->setCompletion(completion, true);
1134 EditableDenseThreeDimensionalModel *model = 1158 }
1135 getConformingOutput<EditableDenseThreeDimensionalModel>(n); 1159 }
1136 if (!model) return; 1160
1137 if (model->isAbandoning()) abandon(); 1161 if (!found) {
1138 model->setCompletion(completion, true); //!!!m_context.updates); 1162 auto model = ModelById::getAs<EditableDenseThreeDimensionalModel>(outputId);
1139 } 1163 if (model) {
1140 } 1164 found = true;
1141 1165 model->setCompletion(completion, true);
1166 }
1167 }
1168 }
1169