Chris@0: 
Chris@0: /* -*- c-basic-offset: 4 -*-  vi:set ts=8 sts=4 sw=4: */
Chris@0: 
Chris@0: /*
Chris@0:     A waveform viewer and audio annotation editor.
Chris@2:     Chris Cannam, Queen Mary University of London, 2005-2006
Chris@0:     
Chris@0:     This is experimental software.  Not for distribution.
Chris@0: */
Chris@0: 
Chris@0: #include "FeatureExtractionPluginTransform.h"
Chris@0: 
Chris@0: #include "plugin/FeatureExtractionPluginFactory.h"
Chris@0: #include "plugin/FeatureExtractionPlugin.h"
Chris@0: 
Chris@0: #include "base/Model.h"
Chris@0: #include "model/SparseOneDimensionalModel.h"
Chris@0: #include "model/SparseTimeValueModel.h"
Chris@0: #include "model/DenseThreeDimensionalModel.h"
Chris@0: #include "model/DenseTimeValueModel.h"
Chris@0: 
Chris@0: #include <iostream>
Chris@0: 
Chris@0: FeatureExtractionPluginTransform::FeatureExtractionPluginTransform(Model *inputModel,
Chris@0: 								   QString pluginId,
Chris@0: 								   QString outputName) :
Chris@0:     Transform(inputModel),
Chris@0:     m_plugin(0),
Chris@0:     m_descriptor(0),
Chris@0:     m_outputFeatureNo(0)
Chris@0: {
Chris@0:     std::cerr << "FeatureExtractionPluginTransform::FeatureExtractionPluginTransform: plugin " << pluginId.toStdString() << ", outputName " << outputName.toStdString() << std::endl;
Chris@0: 
Chris@0:     FeatureExtractionPluginFactory *factory =
Chris@0: 	FeatureExtractionPluginFactory::instanceFor(pluginId);
Chris@0: 
Chris@0:     if (!factory) {
Chris@0: 	std::cerr << "FeatureExtractionPluginTransform: No factory available for plugin id \""
Chris@0: 		  << pluginId.toStdString() << "\"" << std::endl;
Chris@0: 	return;
Chris@0:     }
Chris@0: 
Chris@0:     m_plugin = factory->instantiatePlugin(pluginId, m_input->getSampleRate());
Chris@0: 
Chris@0:     if (!m_plugin) {
Chris@0: 	std::cerr << "FeatureExtractionPluginTransform: Failed to instantiate plugin \""
Chris@0: 		  << pluginId.toStdString() << "\"" << std::endl;
Chris@0: 	return;
Chris@0:     }
Chris@0: 
Chris@0:     FeatureExtractionPlugin::OutputList outputs =
Chris@0: 	m_plugin->getOutputDescriptors();
Chris@0: 
Chris@0:     if (outputs.empty()) {
Chris@0: 	std::cerr << "FeatureExtractionPluginTransform: Plugin \""
Chris@0: 		  << pluginId.toStdString() << "\" has no outputs" << std::endl;
Chris@0: 	return;
Chris@0:     }
Chris@0:     
Chris@0:     for (size_t i = 0; i < outputs.size(); ++i) {
Chris@0: 	if (outputName == "" || outputs[i].name == outputName.toStdString()) {
Chris@0: 	    m_outputFeatureNo = i;
Chris@0: 	    m_descriptor = new FeatureExtractionPlugin::OutputDescriptor
Chris@0: 		(outputs[i]);
Chris@0: 	    break;
Chris@0: 	}
Chris@0:     }
Chris@0: 
Chris@0:     if (!m_descriptor) {
Chris@0: 	std::cerr << "FeatureExtractionPluginTransform: Plugin \""
Chris@0: 		  << pluginId.toStdString() << "\" has no output named \""
Chris@0: 		  << outputName.toStdString() << "\"" << std::endl;
Chris@0: 	return;
Chris@0:     }
Chris@0: 
Chris@0:     std::cerr << "FeatureExtractionPluginTransform: output sample type "
Chris@0: 	      << m_descriptor->sampleType << std::endl;
Chris@0: 
Chris@0:     int valueCount = 1;
Chris@0:     float minValue = 0.0, maxValue = 0.0;
Chris@0:     
Chris@0:     if (m_descriptor->hasFixedValueCount) {
Chris@0: 	valueCount = m_descriptor->valueCount;
Chris@0:     }
Chris@0: 
Chris@0:     if (valueCount > 0 && m_descriptor->hasKnownExtents) {
Chris@0: 	minValue = m_descriptor->minValue;
Chris@0: 	maxValue = m_descriptor->maxValue;
Chris@0:     }
Chris@0: 
Chris@0:     size_t modelRate = m_input->getSampleRate();
Chris@0:     size_t modelResolution = 1;
Chris@0:     
Chris@0:     switch (m_descriptor->sampleType) {
Chris@0: 
Chris@0:     case FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate:
Chris@0: 	if (m_descriptor->sampleRate != 0.0) {
Chris@0: 	    modelResolution = size_t(modelRate / m_descriptor->sampleRate + 0.001);
Chris@0: 	}
Chris@0: 	break;
Chris@0: 
Chris@0:     case FeatureExtractionPlugin::OutputDescriptor::OneSamplePerStep:
Chris@0: 	modelResolution = m_plugin->getPreferredStepSize();
Chris@0: 	break;
Chris@0: 
Chris@0:     case FeatureExtractionPlugin::OutputDescriptor::FixedSampleRate:
Chris@0: 	modelRate = m_descriptor->sampleRate;
Chris@0: 	break;
Chris@0:     }
Chris@0: 
Chris@0:     if (valueCount == 0) {
Chris@0: 
Chris@0: 	m_output = new SparseOneDimensionalModel(modelRate, modelResolution);
Chris@0: 
Chris@0:     } else if (valueCount == 1 ||
Chris@0: 
Chris@0: 	       // We don't have a sparse 3D model
Chris@0: 	       m_descriptor->sampleType ==
Chris@0: 	       FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0: 	
Chris@0: 	m_output = new SparseTimeValueModel(modelRate, modelResolution,
Chris@0: 					    minValue, maxValue, false);
Chris@0: 
Chris@0:     } else {
Chris@0: 	
Chris@0: 	m_output = new DenseThreeDimensionalModel(modelRate, modelResolution,
Chris@19: 						  valueCount, false);
Chris@0:     }
Chris@0: }
Chris@0: 
Chris@0: FeatureExtractionPluginTransform::~FeatureExtractionPluginTransform()
Chris@0: {
Chris@0:     delete m_plugin;
Chris@0:     delete m_descriptor;
Chris@0: }
Chris@0: 
Chris@0: DenseTimeValueModel *
Chris@0: FeatureExtractionPluginTransform::getInput()
Chris@0: {
Chris@0:     DenseTimeValueModel *dtvm =
Chris@0: 	dynamic_cast<DenseTimeValueModel *>(getInputModel());
Chris@0:     if (!dtvm) {
Chris@0: 	std::cerr << "FeatureExtractionPluginTransform::getInput: WARNING: Input model is not conformable to DenseTimeValueModel" << std::endl;
Chris@0:     }
Chris@0:     return dtvm;
Chris@0: }
Chris@0: 
Chris@0: void
Chris@0: FeatureExtractionPluginTransform::run()
Chris@0: {
Chris@0:     DenseTimeValueModel *input = getInput();
Chris@0:     if (!input) return;
Chris@0: 
Chris@0:     if (!m_output) return;
Chris@0: 
Chris@0:     size_t channelCount = input->getChannelCount();
Chris@0:     if (m_plugin->getMaxChannelCount() < channelCount) {
Chris@0: 	channelCount = 1;
Chris@0:     }
Chris@0:     if (m_plugin->getMinChannelCount() > channelCount) {
Chris@0: 	std::cerr << "FeatureExtractionPluginTransform::run: "
Chris@0: 		  << "Can't provide enough channels to plugin (plugin min "
Chris@0: 		  << m_plugin->getMinChannelCount() << ", max "
Chris@0: 		  << m_plugin->getMaxChannelCount() << ", input model has "
Chris@0: 		  << input->getChannelCount() << ")" << std::endl;
Chris@0: 	return;
Chris@0:     }
Chris@0: 
Chris@0:     size_t sampleRate = m_input->getSampleRate();
Chris@0: 
Chris@0:     size_t stepSize = m_plugin->getPreferredStepSize();
Chris@0:     size_t blockSize = m_plugin->getPreferredBlockSize();
Chris@0: 
Chris@0:     m_plugin->initialise(channelCount, stepSize, blockSize);
Chris@0: 
Chris@0:     float **buffers = new float*[channelCount];
Chris@0:     for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@0: 	buffers[ch] = new float[blockSize];
Chris@0:     }
Chris@0: 
Chris@0:     size_t startFrame = m_input->getStartFrame();
Chris@0:     size_t   endFrame = m_input->getEndFrame();
Chris@0:     size_t blockFrame = startFrame;
Chris@0: 
Chris@0:     size_t prevCompletion = 0;
Chris@0: 
Chris@0:     while (blockFrame < endFrame) {
Chris@0: 
Chris@0: //	std::cerr << "FeatureExtractionPluginTransform::run: blockFrame "
Chris@0: //		  << blockFrame << std::endl;
Chris@0: 
Chris@0: 	size_t completion =
Chris@0: 	    (((blockFrame - startFrame) / stepSize) * 99) /
Chris@0: 	    (   (endFrame - startFrame) / stepSize);
Chris@0: 
Chris@0: 	// channelCount is either m_input->channelCount or 1
Chris@0: 
Chris@0: 	size_t got = 0;
Chris@0: 
Chris@0: 	if (channelCount == 1) {
Chris@0: 	    got = input->getValues
Chris@0: 		(-1, blockFrame, blockFrame + blockSize, buffers[0]);
Chris@0: 	    while (got < blockSize) {
Chris@0: 		buffers[0][got++] = 0.0;
Chris@0: 	    }
Chris@0: 	} else {
Chris@0: 	    for (size_t ch = 0; ch < channelCount; ++ch) {
Chris@0: 		got = input->getValues
Chris@0: 		    (ch, blockFrame, blockFrame + blockSize, buffers[ch]);
Chris@0: 		while (got < blockSize) {
Chris@0: 		    buffers[ch][got++] = 0.0;
Chris@0: 		}
Chris@0: 	    }
Chris@0: 	}
Chris@0: 
Chris@0: 	FeatureExtractionPlugin::FeatureSet features = m_plugin->process
Chris@0: 	    (buffers, RealTime::frame2RealTime(blockFrame, sampleRate));
Chris@0: 
Chris@0: 	for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@0: 	    FeatureExtractionPlugin::Feature feature =
Chris@0: 		features[m_outputFeatureNo][fi];
Chris@0: 	    addFeature(blockFrame, feature);
Chris@0: 	}
Chris@0: 
Chris@0: 	if (blockFrame == startFrame || completion > prevCompletion) {
Chris@0: 	    setCompletion(completion);
Chris@0: 	    prevCompletion = completion;
Chris@0: 	}
Chris@0: 
Chris@0: 	blockFrame += stepSize;
Chris@0:     }
Chris@0: 
Chris@0:     FeatureExtractionPlugin::FeatureSet features = m_plugin->getRemainingFeatures();
Chris@0: 
Chris@0:     for (size_t fi = 0; fi < features[m_outputFeatureNo].size(); ++fi) {
Chris@0: 	FeatureExtractionPlugin::Feature feature =
Chris@0: 	    features[m_outputFeatureNo][fi];
Chris@0: 	addFeature(blockFrame, feature);
Chris@0:     }
Chris@0: 
Chris@0:     setCompletion(100);
Chris@0: }
Chris@0: 
Chris@0: 
Chris@0: void
Chris@0: FeatureExtractionPluginTransform::addFeature(size_t blockFrame,
Chris@0: 					     const FeatureExtractionPlugin::Feature &feature)
Chris@0: {
Chris@0:     size_t inputRate = m_input->getSampleRate();
Chris@0: 
Chris@0: //    std::cerr << "FeatureExtractionPluginTransform::addFeature("
Chris@0: //	      << blockFrame << ")" << std::endl;
Chris@0: 
Chris@0:     int valueCount = 1;
Chris@0:     if (m_descriptor->hasFixedValueCount) {
Chris@0: 	valueCount = m_descriptor->valueCount;
Chris@0:     }
Chris@0: 
Chris@0:     size_t frame = blockFrame;
Chris@0: 
Chris@0:     if (m_descriptor->sampleType ==
Chris@0: 	FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0: 
Chris@0: 	if (!feature.hasTimestamp) {
Chris@0: 	    std::cerr
Chris@0: 		<< "WARNING: FeatureExtractionPluginTransform::addFeature: "
Chris@0: 		<< "Feature has variable sample rate but no timestamp!"
Chris@0: 		<< std::endl;
Chris@0: 	    return;
Chris@0: 	} else {
Chris@0: 	    frame = RealTime::realTime2Frame(feature.timestamp, inputRate);
Chris@0: 	}
Chris@0: 
Chris@0:     } else if (m_descriptor->sampleType ==
Chris@0: 	       FeatureExtractionPlugin::OutputDescriptor::FixedSampleRate) {
Chris@0: 
Chris@0: 	if (feature.hasTimestamp) {
Chris@0: 	    //!!! warning: sampleRate may be non-integral
Chris@0: 	    frame = RealTime::realTime2Frame(feature.timestamp,
Chris@0: 					     m_descriptor->sampleRate);
Chris@0: 	} else {
Chris@0: 	    frame = m_output->getEndFrame() + 1;
Chris@0: 	}
Chris@0:     }
Chris@0: 	
Chris@0:     if (valueCount == 0) {
Chris@0: 
Chris@0: 	SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
Chris@0: 	if (!model) return;
Chris@0: 	model->addPoint(SparseOneDimensionalModel::Point(frame, feature.label.c_str()));
Chris@0: 	
Chris@0:     } else if (valueCount == 1 ||
Chris@0: 	       m_descriptor->sampleType == 
Chris@0: 	       FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0: 
Chris@0: 	float value = 0.0;
Chris@0: 	if (feature.values.size() > 0) value = feature.values[0];
Chris@0: 
Chris@0: 	SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
Chris@0: 	if (!model) return;
Chris@0: 	model->addPoint(SparseTimeValueModel::Point(frame, value, feature.label.c_str()));
Chris@0: 	
Chris@0:     } else {
Chris@0: 	
Chris@0: 	DenseThreeDimensionalModel::BinValueSet values = feature.values;
Chris@0: 	
Chris@0: 	DenseThreeDimensionalModel *model = getOutput<DenseThreeDimensionalModel>();
Chris@0: 	if (!model) return;
Chris@0: 
Chris@0: 	model->setBinValues(frame, values);
Chris@0:     }
Chris@0: }
Chris@0: 
Chris@0: void
Chris@0: FeatureExtractionPluginTransform::setCompletion(int completion)
Chris@0: {
Chris@0:     int valueCount = 1;
Chris@0:     if (m_descriptor->hasFixedValueCount) {
Chris@0: 	valueCount = m_descriptor->valueCount;
Chris@0:     }
Chris@0: 
Chris@0:     if (valueCount == 0) {
Chris@0: 
Chris@0: 	SparseOneDimensionalModel *model = getOutput<SparseOneDimensionalModel>();
Chris@0: 	if (!model) return;
Chris@0: 	model->setCompletion(completion);
Chris@0: 
Chris@0:     } else if (valueCount == 1 ||
Chris@0: 	       m_descriptor->sampleType ==
Chris@0: 	       FeatureExtractionPlugin::OutputDescriptor::VariableSampleRate) {
Chris@0: 
Chris@0: 	SparseTimeValueModel *model = getOutput<SparseTimeValueModel>();
Chris@0: 	if (!model) return;
Chris@0: 	model->setCompletion(completion);
Chris@0: 
Chris@0:     } else {
Chris@0: 
Chris@19: 	DenseThreeDimensionalModel *model = getOutput<DenseThreeDimensionalModel>();
Chris@19: 	if (!model) return;
Chris@19: 	model->setCompletion(completion);
Chris@0:     }
Chris@0: }
Chris@0: