21 #include <vamp-hostsdk/Plugin.h> 50 SVDEBUG <<
"FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: plugin " <<
m_transforms.begin()->getPluginIdentifier() <<
", outputName " <<
m_transforms.begin()->getOutput() << endl;
60 SVDEBUG <<
"FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() <<
" transform(s)" << endl;
62 SVDEBUG <<
"FeatureExtractionModelTransformer::FeatureExtractionModelTransformer: " << transforms.size() <<
" transform(s), first has plugin " <<
m_transforms.begin()->getPluginIdentifier() <<
", outputName " <<
m_transforms.begin()->getOutput() << endl;
88 m_message = tr(
"Transforms supplied to a single FeatureExtractionModelTransformer instance must be similar in every respect except plugin output");
102 m_message = tr(
"No factory available for feature extraction plugin id \"%1\" (unknown plugin type, or internal error?)").arg(pluginId);
107 auto input = ModelById::getAs<DenseTimeValueModel>(
getInputModel());
109 m_message = tr(
"Input model for feature extraction plugin \"%1\" is of wrong type (internal error?)").arg(pluginId);
114 SVDEBUG <<
"FeatureExtractionModelTransformer: Instantiating plugin for transform in thread " 115 << QThread::currentThreadId() << endl;
119 m_message = tr(
"Failed to instantiate plugin \"%1\"").arg(pluginId);
130 int channelCount = input->getChannelCount();
131 if ((
int)
m_plugin->getMaxChannelCount() < channelCount) {
134 if ((
int)
m_plugin->getMinChannelCount() > channelCount) {
135 m_message = tr(
"Cannot provide enough channels to feature extraction plugin \"%1\" (plugin min is %2, max %3; input model has %4)")
137 .arg(
m_plugin->getMinChannelCount())
138 .arg(
m_plugin->getMaxChannelCount())
139 .arg(input->getChannelCount());
147 SVDEBUG <<
"Initialising feature extraction plugin with channels = " 148 << channelCount <<
", step = " << step
149 <<
", block = " << block << endl;
151 if (!
m_plugin->initialise(channelCount, step, block)) {
153 int preferredStep = int(
m_plugin->getPreferredStepSize());
154 int preferredBlock = int(
m_plugin->getPreferredBlockSize());
156 if (step != preferredStep || block != preferredBlock) {
158 SVDEBUG <<
"Initialisation failed, trying again with preferred step = " 159 << preferredStep <<
", block = " << preferredBlock << endl;
161 if (!
m_plugin->initialise(channelCount,
165 SVDEBUG <<
"Initialisation failed again" << endl;
167 m_message = tr(
"Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
173 SVDEBUG <<
"Initialisation succeeded this time" << endl;
179 m_message = tr(
"Feature extraction plugin \"%1\" rejected the given step and block sizes (%2 and %3); using plugin defaults (%4 and %5) instead")
184 .arg(preferredBlock);
190 SVDEBUG <<
"Initialisation failed (with step = " << step
191 <<
" and block = " << block
192 <<
", both matching the plugin's preference)" << endl;
194 m_message = tr(
"Failed to initialise feature extraction plugin \"%1\"").arg(pluginId);
199 SVDEBUG <<
"Initialisation succeeded" << endl;
203 QString pv = QString(
"%1").arg(
m_plugin->getPluginVersion());
205 QString vm = tr(
"Transform was configured for version %1 of plugin \"%2\", but the plugin being used is version %3")
218 Vamp::Plugin::OutputList outputs =
m_plugin->getOutputDescriptors();
220 if (outputs.empty()) {
221 m_message = tr(
"Plugin \"%1\" has no outputs").arg(pluginId);
231 outputs[i].identifier ==
242 m_message = tr(
"Plugin \"%1\" has no output named \"%2\"")
266 SVDEBUG <<
"FeatureExtractionModelTransformer: deleting plugin for transform in thread " 267 << QThread::currentThreadId() << endl;
272 }
catch (
const std::exception &e) {
276 SVCERR <<
"FeatureExtractionModelTransformer: caught exception while deleting plugin: " << e.what() << endl;
286 auto input = ModelById::getAs<DenseTimeValueModel>(
getInputModel());
293 float minValue = 0.0, maxValue = 0.0;
294 bool haveExtents =
false;
311 int modelResolution = 1;
314 Vamp::Plugin::OutputDescriptor::OneSamplePerStep) {
323 if (outputRate > input->getSampleRate()) {
324 SVDEBUG <<
"WARNING: plugin reports output sample rate as " 326 <<
" (can't display features with finer resolution than the input rate of " 327 << modelRate <<
")" << endl;
328 outputRate = modelRate;
334 case Vamp::Plugin::OutputDescriptor::VariableSampleRate:
335 if (outputRate != 0.0) {
336 modelResolution = int(round(modelRate / outputRate));
340 case Vamp::Plugin::OutputDescriptor::OneSamplePerStep:
344 case Vamp::Plugin::OutputDescriptor::FixedSampleRate:
345 if (outputRate <= 0.0) {
346 SVDEBUG <<
"WARNING: Fixed sample-rate plugin reports invalid sample rate " <<
m_descriptors[n].sampleRate <<
"; defaulting to input rate of " << input->getSampleRate() << endl;
349 modelResolution = int(round(modelRate / outputRate));
355 bool preDurationPlugin = (
m_plugin->getVampApiVersion() < 2);
357 std::shared_ptr<Model> out;
364 SVDEBUG <<
"FeatureExtractionModelTransformer::createOutputModels: " 365 <<
"creating a SparseOneDimensionalModel" << endl;
367 out = std::make_shared<SparseOneDimensionalModel>
368 (modelRate, modelResolution,
false);
371 out->setRDFTypeURI(outputEventTypeURI);
373 }
else if ((preDurationPlugin && binCount > 1 &&
375 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) ||
398 bool isNoteModel =
false;
403 if (binCount > 1) isNoteModel =
true;
422 settings.beginGroup(
"Transformer");
423 bool flexi = settings.value(
"use-flexi-note-model",
false).toBool();
426 SVDEBUG <<
"FeatureExtractionModelTransformer::createOutputModels: " 427 <<
"creating a NoteModel (flexi = " << flexi <<
")" << endl;
432 (modelRate, modelResolution, minValue, maxValue,
false,
436 (modelRate, modelResolution,
false,
444 SVDEBUG <<
"FeatureExtractionModelTransformer::createOutputModels: " 445 <<
"creating a RegionModel" << endl;
450 (modelRate, modelResolution, minValue, maxValue,
false);
453 (modelRate, modelResolution,
false);
460 out->setRDFTypeURI(outputEventTypeURI);
462 }
else if (binCount == 1 ||
464 Vamp::Plugin::OutputDescriptor::VariableSampleRate)) {
488 if (!haveBinCount || binCount > 1) {
492 SVDEBUG <<
"FeatureExtractionModelTransformer::createOutputModels: " 493 <<
"creating a SparseTimeValueModel " 494 <<
"(additional models to come? -> " 500 (modelRate, modelResolution, minValue, maxValue,
false);
503 (modelRate, modelResolution,
false);
506 Vamp::Plugin::OutputList outputs =
m_plugin->getOutputDescriptors();
512 out->setRDFTypeURI(outputEventTypeURI);
520 SVDEBUG <<
"FeatureExtractionModelTransformer::createOutputModels: " 521 <<
"creating a BasicCompressedDenseThreeDimensionalModel" 526 (modelRate, modelResolution, binCount,
false);
529 std::vector<QString> names;
530 for (
int i = 0; i < (int)
m_descriptors[n].binNames.size(); ++i) {
533 model->setBinNames(names);
539 out->setRDFTypeURI(outputSignalTypeURI);
570 for (
auto m: mp.second) {
571 mm.push_back(m.second);
581 if (p.second)
return true;
590 SVCERR <<
"Internal error: binNo == 0 in getAdditionalModel (should be using primary model, not calling getAdditionalModel)" << endl;
595 SVCERR <<
"getAdditionalModel: Output " << n <<
" out of range" << endl;
608 SVDEBUG <<
"getAdditionalModel(" << n <<
", " << binNo
609 <<
"): creating" << endl;
611 auto baseModel = ModelById::getAs<SparseTimeValueModel>(
m_outputs[n]);
613 SVCERR <<
"getAdditionalModel: Output model not conformable, or has vanished" << endl;
617 SVDEBUG <<
"getAdditionalModel(" << n <<
", " << binNo
618 <<
"): (from " << baseModel <<
")" << endl;
622 baseModel->getResolution(),
623 baseModel->getValueMinimum(),
624 baseModel->getValueMaximum(),
631 (std::shared_ptr<SparseTimeValueModel>(additional));
644 }
catch (
const std::exception &e) {
662 auto input = ModelById::getAs<DenseTimeValueModel>(inputId);
663 if (!input || !input->isOK()) {
667 ready = input->isReady();
670 SVDEBUG <<
"FeatureExtractionModelTransformer::run: Waiting for input model " 671 << inputId <<
" to be ready..." << endl;
677 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 678 SVDEBUG <<
"FeatureExtractionModelTransformer::run: Input model " 679 << inputId <<
" is ready, going ahead" << endl;
689 auto input = ModelById::getAs<DenseTimeValueModel>(inputId);
695 sampleRate = input->getSampleRate();
697 channelCount = input->getChannelCount();
698 if ((
int)
m_plugin->getMaxChannelCount() < channelCount) {
702 startFrame = input->getStartFrame();
703 endFrame = input->getEndFrame();
706 float **buffers =
new float*[channelCount];
707 for (
int ch = 0; ch < channelCount; ++ch) {
708 buffers[ch] =
new float[primaryTransform.
getBlockSize() + 2];
714 bool frequencyDomain = (
m_plugin->getInputDomain() ==
715 Vamp::Plugin::FrequencyDomain);
717 std::vector<FFTModel *> fftModels;
719 if (frequencyDomain) {
720 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 721 SVDEBUG <<
"FeatureExtractionModelTransformer::run: Input is frequency-domain" << endl;
723 for (
int ch = 0; ch < channelCount; ++ch) {
737 SVDEBUG <<
"FeatureExtractionModelTransformer::run: Failed to create FFT model for input model " << inputId <<
": " << err << endl;
738 m_message =
"Failed to create the FFT model for this feature extraction model transformer: error is: " + err;
739 for (
int cch = 0; cch < ch; ++cch) {
740 delete fftModels[cch];
745 fftModels.push_back(model);
747 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 748 SVDEBUG <<
"FeatureExtractionModelTransformer::run: Created FFT model(s) for frequency-domain input" << endl;
761 if (contextStart == 0 || contextStart < startFrame) {
762 contextStart = startFrame;
765 if (contextDuration == 0) {
766 contextDuration = endFrame - contextStart;
768 if (contextStart + contextDuration > endFrame) {
769 contextDuration = endFrame - contextStart;
774 long prevCompletion = 0;
780 float *reals =
nullptr;
781 float *imaginaries =
nullptr;
782 if (frequencyDomain) {
783 reals =
new float[blockSize/2 + 1];
784 imaginaries =
new float[blockSize/2 + 1];
792 if (frequencyDomain) {
793 if (blockFrame -
int(blockSize)/2 >
794 contextStart + contextDuration) {
799 contextStart + contextDuration) {
804 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 805 SVDEBUG <<
"FeatureExtractionModelTransformer::run: blockFrame " 806 << blockFrame <<
", endFrame " << endFrame <<
", blockSize " 807 << blockSize << endl;
811 ((((blockFrame - contextStart) / stepSize) * 99) /
812 (contextDuration / stepSize + 1));
814 bool haveAllModels =
true;
816 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 817 SVDEBUG <<
"FeatureExtractionModelTransformer::run: Input model " << inputId <<
" no longer exists" << endl;
819 haveAllModels =
false;
821 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 822 SVDEBUG <<
"Input model " << inputId <<
" still exists" << endl;
827 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 828 SVDEBUG <<
"FeatureExtractionModelTransformer::run: Output model " << mid <<
" no longer exists" << endl;
830 haveAllModels =
false;
832 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 833 SVDEBUG <<
"Output model " << mid <<
" still exists" << endl;
837 if (!haveAllModels) {
842 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 843 SVDEBUG <<
"FeatureExtractionModelTransformer::run: All models still exist" << endl;
848 if (frequencyDomain) {
849 for (
int ch = 0; ch < channelCount; ++ch) {
850 int column = int((blockFrame - startFrame) / stepSize);
851 if (fftModels[ch]->getValuesAt(column, reals, imaginaries)) {
852 for (
int i = 0; i <= blockSize/2; ++i) {
853 buffers[ch][i*2] = reals[i];
854 buffers[ch][i*2+1] = imaginaries[i];
857 for (
int i = 0; i <= blockSize/2; ++i) {
858 buffers[ch][i*2] = 0.f;
859 buffers[ch][i*2+1] = 0.f;
863 error = fftModels[ch]->getError();
865 SVCERR <<
"FeatureExtractionModelTransformer::run: Abandoning, error is " << error << endl;
872 getFrames(channelCount, blockFrame, blockSize, buffers);
891 if (blockFrame == contextStart || completion > prevCompletion) {
895 prevCompletion = completion;
898 blockFrame += stepSize;
903 auto features =
m_plugin->getRemainingFeatures();
915 }
catch (
const std::exception &e) {
916 SVCERR <<
"FeatureExtractionModelTransformer::run: Exception caught: " 922 for (
int j = 0; j < (int)
m_outputNos.size(); ++j) {
926 if (frequencyDomain) {
927 for (
int ch = 0; ch < channelCount; ++ch) {
928 delete fftModels[ch];
931 delete[] imaginaries;
934 for (
int ch = 0; ch < channelCount; ++ch) {
935 delete[] buffers[ch];
950 if (startFrame < 0) {
951 for (
int c = 0; c < channelCount; ++c) {
952 for (
sv_frame_t i = 0; i < size && startFrame + i < 0; ++i) {
953 buffers[c][i] = 0.0f;
956 offset = -startFrame;
958 if (size <= 0)
return;
962 auto input = ModelById::getAs<DenseTimeValueModel>(
getInputModel());
969 if (channelCount == 1) {
974 copy(data.begin(), data.end(), buffers[0] + offset);
978 float cc = float(input->getChannelCount());
980 buffers[0][i + offset] /= cc;
986 auto data = input->getMultiChannelData(0, channelCount-1, startFrame, size);
988 got = data[0].size();
990 copy(data[c].begin(), data[c].end(), buffers[c] + offset);
996 for (
int c = 0; c < channelCount; ++c) {
997 buffers[c][got + offset] = 0.0;
1006 const Vamp::Plugin::Feature &feature)
1022 Vamp::Plugin::OutputDescriptor::VariableSampleRate) {
1024 if (!feature.hasTimestamp) {
1026 <<
"WARNING: FeatureExtractionModelTransformer::addFeature: " 1027 <<
"Feature has variable sample rate but no timestamp!" 1038 Vamp::Plugin::OutputDescriptor::FixedSampleRate) {
1045 if (!feature.hasTimestamp) {
1048 RealTime ts(feature.timestamp.sec, feature.timestamp.nsec);
1062 <<
"WARNING: FeatureExtractionModelTransformer::addFeature: " 1063 <<
"Negative frame counts are not supported (frame = " << frame
1064 <<
" from timestamp " << feature.timestamp
1065 <<
"), dropping feature" 1077 if (isOutputType<SparseOneDimensionalModel>(n)) {
1079 auto model = ModelById::getAs<SparseOneDimensionalModel>(outputId);
1081 model->add(
Event(frame, feature.label.c_str()));
1083 }
else if (isOutputType<SparseTimeValueModel>(n)) {
1085 auto model = ModelById::getAs<SparseTimeValueModel>(outputId);
1088 for (
int i = 0;
in_range_for(feature.values, i); ++i) {
1090 float value = feature.values[i];
1092 QString label = feature.label.c_str();
1093 if (feature.values.size() > 1) {
1094 label = QString(
"[%1] %2").arg(i+1).arg(label);
1097 auto targetModel = model;
1100 targetModel = ModelById::getAs<SparseTimeValueModel>
1102 if (!targetModel) targetModel = model;
1105 targetModel->add(
Event(frame, value, label));
1108 }
else if (isOutputType<NoteModel>(n) || isOutputType<RegionModel>(n)) {
1113 if ((
int)feature.values.size() > index) {
1114 value = feature.values[index++];
1118 if (feature.hasDuration) {
1122 duration = lrintf(feature.values[index++]);
1126 auto noteModel = ModelById::getAs<NoteModel>(outputId);
1129 float velocity = 100;
1130 if ((
int)feature.values.size() > index) {
1131 velocity = feature.values[index++];
1133 if (velocity < 0) velocity = 127;
1134 if (velocity > 127) velocity = 127;
1136 noteModel->add(
Event(frame, value,
1139 feature.label.c_str()));
1142 auto regionModel = ModelById::getAs<RegionModel>(outputId);
1145 if (feature.hasDuration && !feature.values.empty()) {
1147 for (
int i = 0;
in_range_for(feature.values, i); ++i) {
1149 float value = feature.values[i];
1151 QString label = feature.label.c_str();
1152 if (feature.values.size() > 1) {
1153 label = QString(
"[%1] %2").arg(i+1).arg(label);
1156 regionModel->add(
Event(frame,
1163 regionModel->add(
Event(frame,
1166 feature.label.c_str()));
1170 }
else if (isOutputType<BasicCompressedDenseThreeDimensionalModel>(n)) {
1177 values.insert(values.begin(),
1178 feature.values.begin(), feature.values.end());
1183 model->setColumn(
int(frame / model->getResolution()), values);
1187 SVDEBUG <<
"FeatureExtractionModelTransformer::addFeature: Unknown output model type - possibly a deleted model" << endl;
1195 #ifdef DEBUG_FEATURE_EXTRACTION_TRANSFORMER_RUN 1196 SVDEBUG <<
"FeatureExtractionModelTransformer::setCompletion(" 1197 << completion <<
")" << endl;
1201 (setOutputCompletion<SparseOneDimensionalModel>(n, completion) ||
1202 setOutputCompletion<SparseTimeValueModel>(n, completion) ||
1203 setOutputCompletion<NoteModel>(n, completion) ||
1204 setOutputCompletion<RegionModel>(n, completion) ||
1205 setOutputCompletion<BasicCompressedDenseThreeDimensionalModel>(n, completion));
double sv_samplerate_t
Sample rate.
bool isOK() const override
Return true if the model was constructed successfully.
int64_t sv_frame_t
Frame index, the unit of our time axis.
static RealTime frame2RealTime(sv_frame_t frame, sv_samplerate_t sampleRate)
Convert a sample frame at the given sample rate into a RealTime.
static std::shared_ptr< Derived > getAs(Id id)
void setRDFTypeURI(QString uri)
Set the event, feature, or signal type URI for the features contained in this model, according to the Audio Features RDF ontology.
An implementation of DenseThreeDimensionalModel that makes FFT data derived from a DenseTimeValueMode...
QString getOutputSignalTypeURI(QString outputId) const
static Id add(std::shared_ptr< Item > item)
QString getOutputEventTypeURI(QString outputId) const
static sv_frame_t realTime2Frame(const RealTime &r, sv_samplerate_t sampleRate)
Convert a RealTime into a sample frame at the given sample rate.
A model representing a wiggly-line plot with points at arbitrary intervals of the model resolution...
RegionModel – a model for intervals associated with a value, which we call regions for no very compe...
bool in_range_for(const C &container, T i)
Check whether an integer index is in range for a container, avoiding overflows and signed/unsigned co...
void setScaleUnits(QString units)
void setScaleUnits(QString units)
An immutable(-ish) type used for point and event representation in sparse models, as well as for inte...
void setScaleUnits(QString units)
virtual void setColumn(int x, const Column &values)
Set the entire set of bin values at the given column.
static std::shared_ptr< Item > get(Id id)
RealTime represents time values to nanosecond precision with accurate arithmetic and frame-rate conve...