Chris@767: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*-  vi:set ts=8 sts=4 sw=4: */
Chris@767: 
Chris@767: /*
Chris@767:     Sonic Visualiser
Chris@767:     An audio file viewer and annotation editor.
Chris@767:     Centre for Digital Music, Queen Mary, University of London.
Chris@767:     
Chris@767:     This program is free software; you can redistribute it and/or
Chris@767:     modify it under the terms of the GNU General Public License as
Chris@767:     published by the Free Software Foundation; either version 2 of the
Chris@767:     License, or (at your option) any later version.  See the file
Chris@767:     COPYING included with this distribution for more information.
Chris@767: */
Chris@767: 
Chris@767: #include "TransformDTWAligner.h"
Chris@767: #include "DTW.h"
Chris@767: 
Chris@767: #include "data/model/SparseTimeValueModel.h"
Chris@771: #include "data/model/NoteModel.h"
Chris@767: #include "data/model/RangeSummarisableTimeValueModel.h"
Chris@767: #include "data/model/AlignmentModel.h"
Chris@767: #include "data/model/AggregateWaveModel.h"
Chris@767: 
Chris@767: #include "framework/Document.h"
Chris@767: 
Chris@767: #include "transform/ModelTransformerFactory.h"
Chris@767: #include "transform/FeatureExtractionModelTransformer.h"
Chris@767: 
Chris@767: #include <QSettings>
Chris@767: #include <QMutex>
Chris@767: #include <QMutexLocker>
Chris@767: 
Chris@767: using std::vector;
Chris@767: 
Chris@771: static
Chris@771: TransformDTWAligner::MagnitudePreprocessor identityMagnitudePreprocessor =
Chris@771:     [](double x) {
Chris@771:         return x;
Chris@771:     };
Chris@771: 
Chris@771: static
Chris@771: TransformDTWAligner::RiseFallPreprocessor identityRiseFallPreprocessor =
Chris@771:     [](double prev, double curr) {
Chris@771:         if (prev == curr) {
Chris@771:             return RiseFallDTW::Value({ RiseFallDTW::Direction::None, 0.0 });
Chris@771:         } else if (curr > prev) {
Chris@771:             return RiseFallDTW::Value({ RiseFallDTW::Direction::Up, curr - prev });
Chris@771:         } else {
Chris@771:             return RiseFallDTW::Value({ RiseFallDTW::Direction::Down, prev - curr });
Chris@771:         }
Chris@771:     };
Chris@771: 
Chris@771: QMutex
Chris@771: TransformDTWAligner::m_dtwMutex;
Chris@771: 
Chris@767: TransformDTWAligner::TransformDTWAligner(Document *doc,
Chris@767:                                          ModelId reference,
Chris@767:                                          ModelId toAlign,
Chris@767:                                          Transform transform,
Chris@767:                                          DTWType dtwType) :
Chris@767:     m_document(doc),
Chris@767:     m_reference(reference),
Chris@767:     m_toAlign(toAlign),
Chris@767:     m_transform(transform),
Chris@767:     m_dtwType(dtwType),
Chris@768:     m_incomplete(true),
Chris@771:     m_magnitudePreprocessor(identityMagnitudePreprocessor),
Chris@771:     m_riseFallPreprocessor(identityRiseFallPreprocessor)
Chris@768: {
Chris@768: }
Chris@768: 
Chris@768: TransformDTWAligner::TransformDTWAligner(Document *doc,
Chris@768:                                          ModelId reference,
Chris@768:                                          ModelId toAlign,
Chris@768:                                          Transform transform,
Chris@771:                                          MagnitudePreprocessor outputPreprocessor) :
Chris@768:     m_document(doc),
Chris@768:     m_reference(reference),
Chris@768:     m_toAlign(toAlign),
Chris@768:     m_transform(transform),
Chris@771:     m_dtwType(Magnitude),
Chris@768:     m_incomplete(true),
Chris@771:     m_magnitudePreprocessor(outputPreprocessor),
Chris@771:     m_riseFallPreprocessor(identityRiseFallPreprocessor)
Chris@771: {
Chris@771: }
Chris@771: 
Chris@771: TransformDTWAligner::TransformDTWAligner(Document *doc,
Chris@771:                                          ModelId reference,
Chris@771:                                          ModelId toAlign,
Chris@771:                                          Transform transform,
Chris@771:                                          RiseFallPreprocessor outputPreprocessor) :
Chris@771:     m_document(doc),
Chris@771:     m_reference(reference),
Chris@771:     m_toAlign(toAlign),
Chris@771:     m_transform(transform),
Chris@771:     m_dtwType(RiseFall),
Chris@771:     m_incomplete(true),
Chris@771:     m_magnitudePreprocessor(identityMagnitudePreprocessor),
Chris@771:     m_riseFallPreprocessor(outputPreprocessor)
Chris@767: {
Chris@767: }
Chris@767: 
Chris@767: TransformDTWAligner::~TransformDTWAligner()
Chris@767: {
Chris@767:     if (m_incomplete) {
Chris@767:         if (auto toAlign = ModelById::get(m_toAlign)) {
Chris@767:             toAlign->setAlignment({});
Chris@767:         }
Chris@767:     }
Chris@767:     
Chris@767:     ModelById::release(m_referenceOutputModel);
Chris@767:     ModelById::release(m_toAlignOutputModel);
Chris@767: }
Chris@767: 
Chris@767: bool
Chris@767: TransformDTWAligner::isAvailable()
Chris@767: {
Chris@767:     //!!! needs to be isAvailable(QString transformId)?
Chris@767:     return true;
Chris@767: }
Chris@767: 
Chris@767: void
Chris@767: TransformDTWAligner::begin()
Chris@767: {
Chris@767:     auto reference =
Chris@767:         ModelById::getAs<RangeSummarisableTimeValueModel>(m_reference);
Chris@767:     auto toAlign =
Chris@767:         ModelById::getAs<RangeSummarisableTimeValueModel>(m_toAlign);
Chris@767: 
Chris@767:     if (!reference || !toAlign) return;
Chris@767: 
Chris@767:     SVCERR << "TransformDTWAligner[" << this << "]: begin(): aligning "
Chris@767:            << m_toAlign << " against reference " << m_reference << endl;
Chris@767:     
Chris@767:     ModelTransformerFactory *mtf = ModelTransformerFactory::getInstance();
Chris@767: 
Chris@767:     QString message;
Chris@767: 
Chris@767:     m_referenceOutputModel = mtf->transform(m_transform, m_reference, message);
Chris@767:     auto referenceOutputModel = ModelById::get(m_referenceOutputModel);
Chris@767:     if (!referenceOutputModel) {
Chris@767:         SVCERR << "Align::alignModel: ERROR: Failed to create reference output model (no plugin?)" << endl;
Chris@767:         emit failed(m_toAlign, message);
Chris@767:         return;
Chris@767:     }
Chris@767: 
Chris@767:     SVCERR << "TransformDTWAligner[" << this << "]: begin(): transform id "
Chris@767:            << m_transform.getIdentifier()
Chris@767:            << " is running on reference model" << endl;
Chris@767: 
Chris@767:     message = "";
Chris@767: 
Chris@767:     m_toAlignOutputModel = mtf->transform(m_transform, m_toAlign, message);
Chris@767:     auto toAlignOutputModel = ModelById::get(m_toAlignOutputModel);
Chris@767:     if (!toAlignOutputModel) {
Chris@767:         SVCERR << "Align::alignModel: ERROR: Failed to create toAlign output model (no plugin?)" << endl;
Chris@767:         emit failed(m_toAlign, message);
Chris@767:         return;
Chris@767:     }
Chris@767: 
Chris@767:     SVCERR << "TransformDTWAligner[" << this << "]: begin(): transform id "
Chris@767:            << m_transform.getIdentifier()
Chris@767:            << " is running on toAlign model" << endl;
Chris@767: 
Chris@767:     connect(referenceOutputModel.get(), SIGNAL(completionChanged(ModelId)),
Chris@767:             this, SLOT(completionChanged(ModelId)));
Chris@767:     connect(toAlignOutputModel.get(), SIGNAL(completionChanged(ModelId)),
Chris@767:             this, SLOT(completionChanged(ModelId)));
Chris@767:     
Chris@767:     auto alignmentModel = std::make_shared<AlignmentModel>
Chris@768:         (m_reference, m_toAlign, ModelId());
Chris@767:     m_alignmentModel = ModelById::add(alignmentModel);
Chris@767:     
Chris@767:     toAlign->setAlignment(m_alignmentModel);
Chris@767:     m_document->addNonDerivedModel(m_alignmentModel);
Chris@767: 
Chris@767:     // we wouldn't normally expect these to be true here, but...
Chris@767:     int completion = 0;
Chris@767:     if (referenceOutputModel->isReady(&completion) &&
Chris@767:         toAlignOutputModel->isReady(&completion)) {
Chris@767:         SVCERR << "TransformDTWAligner[" << this << "]: begin(): output models "
Chris@767:                << "are ready already! calling performAlignment" << endl;
Chris@767:         if (performAlignment()) {
Chris@767:             emit complete(m_alignmentModel);
Chris@767:         } else {
Chris@767:             emit failed(m_toAlign, tr("Failed to calculate alignment using DTW"));
Chris@767:         }
Chris@767:     }
Chris@767: }
Chris@767: 
Chris@767: void
Chris@767: TransformDTWAligner::completionChanged(ModelId id)
Chris@767: {
Chris@767:     if (!m_incomplete) {
Chris@767:         return;
Chris@767:     }
Chris@771: /*
Chris@767:     SVCERR << "TransformDTWAligner[" << this << "]: completionChanged: "
Chris@767:            << "model " << id << endl;
Chris@771: */
Chris@767:     auto referenceOutputModel = ModelById::get(m_referenceOutputModel);
Chris@767:     auto toAlignOutputModel = ModelById::get(m_toAlignOutputModel);
Chris@768:     auto alignmentModel = ModelById::getAs<AlignmentModel>(m_alignmentModel);
Chris@767: 
Chris@768:     if (!referenceOutputModel || !toAlignOutputModel || !alignmentModel) {
Chris@767:         return;
Chris@767:     }
Chris@767: 
Chris@767:     int referenceCompletion = 0, toAlignCompletion = 0;
Chris@767:     bool referenceReady = referenceOutputModel->isReady(&referenceCompletion);
Chris@767:     bool toAlignReady = toAlignOutputModel->isReady(&toAlignCompletion);
Chris@767: 
Chris@767:     if (referenceReady && toAlignReady) {
Chris@767: 
Chris@767:         SVCERR << "TransformDTWAligner[" << this << "]: completionChanged: "
Chris@771:                << "both models ready, calling performAlignment" << endl;
Chris@767: 
Chris@768:         alignmentModel->setCompletion(95);
Chris@767:         
Chris@767:         if (performAlignment()) {
Chris@767:             emit complete(m_alignmentModel);
Chris@767:         } else {
Chris@767:             emit failed(m_toAlign, tr("Alignment of transform outputs failed"));
Chris@767:         }
Chris@767: 
Chris@767:     } else {
Chris@771: /*
Chris@767:         SVCERR << "TransformDTWAligner[" << this << "]: completionChanged: "
Chris@767:                << "not ready yet: reference completion " << referenceCompletion
Chris@767:                << ", toAlign completion " << toAlignCompletion << endl;
Chris@771: */
Chris@768:         int completion = std::min(referenceCompletion,
Chris@768:                                   toAlignCompletion);
Chris@768:         completion = (completion * 94) / 100;
Chris@768:         alignmentModel->setCompletion(completion);
Chris@767:     }
Chris@767: }
Chris@767: 
Chris@767: bool
Chris@767: TransformDTWAligner::performAlignment()
Chris@767: {
Chris@767:     if (m_dtwType == Magnitude) {
Chris@767:         return performAlignmentMagnitude();
Chris@767:     } else {
Chris@767:         return performAlignmentRiseFall();
Chris@767:     }
Chris@767: }
Chris@767: 
Chris@767: bool
Chris@771: TransformDTWAligner::getValuesFrom(ModelId modelId,
Chris@771:                                    vector<sv_frame_t> &frames,
Chris@771:                                    vector<double> &values,
Chris@771:                                    sv_frame_t &resolution)
Chris@767: {
Chris@771:     EventVector events;
Chris@767: 
Chris@771:     if (auto model = ModelById::getAs<SparseTimeValueModel>(modelId)) {
Chris@771:         resolution = model->getResolution();
Chris@771:         events = model->getAllEvents();
Chris@771:     } else if (auto model = ModelById::getAs<NoteModel>(modelId)) {
Chris@771:         resolution = model->getResolution();
Chris@771:         events = model->getAllEvents();
Chris@771:     } else {
Chris@771:         SVCERR << "TransformDTWAligner::getValuesFrom: Type of model "
Chris@771:                << modelId << " is not supported" << endl;
Chris@767:         return false;
Chris@767:     }
Chris@767: 
Chris@771:     frames.clear();
Chris@771:     values.clear();
Chris@771: 
Chris@771:     for (auto e: events) {
Chris@771:         frames.push_back(e.getFrame());
Chris@771:         values.push_back(e.getValue());
Chris@771:     }
Chris@771: 
Chris@771:     return true;
Chris@771: }
Chris@771: 
Chris@771: Path
Chris@771: TransformDTWAligner::makePath(const vector<size_t> &alignment,
Chris@771:                               const vector<sv_frame_t> &refFrames,
Chris@771:                               const vector<sv_frame_t> &otherFrames,
Chris@771:                               sv_samplerate_t sampleRate,
Chris@771:                               sv_frame_t resolution)
Chris@771: {
Chris@771:     Path path(sampleRate, resolution);
Chris@771: 
Chris@771:     for (int i = 0; in_range_for(alignment, i); ++i) {
Chris@771: 
Chris@771:         // DTW returns "the index into s2 for each element in s1"
Chris@771:         sv_frame_t refFrame = refFrames[i];
Chris@771: 
Chris@771:         if (!in_range_for(otherFrames, alignment[i])) {
Chris@771:             SVCERR << "TransformDTWAligner::makePath: Internal error: "
Chris@771:                    << "DTW maps index " << i << " in reference frame vector "
Chris@771:                    << "(size " << refFrames.size() << ") onto index "
Chris@771:                    << alignment[i] << " in other frame vector "
Chris@771:                    << "(only size " << otherFrames.size() << ")" << endl;
Chris@771:             continue;
Chris@771:         }
Chris@771:             
Chris@771:         sv_frame_t alignedFrame = otherFrames[alignment[i]];
Chris@771:         path.add(PathPoint(alignedFrame, refFrame));
Chris@771:     }
Chris@771: 
Chris@771:     return path;
Chris@771: }
Chris@771: 
Chris@771: bool
Chris@771: TransformDTWAligner::performAlignmentMagnitude()
Chris@771: {
Chris@771:     auto alignmentModel = ModelById::getAs<AlignmentModel>(m_alignmentModel);
Chris@767:     if (!alignmentModel) {
Chris@767:         return false;
Chris@767:     }
Chris@771: 
Chris@771:     vector<sv_frame_t> refFrames, otherFrames;
Chris@771:     vector<double> refValues, otherValues;
Chris@771:     sv_frame_t resolution = 0;
Chris@771: 
Chris@771:     if (!getValuesFrom(m_referenceOutputModel,
Chris@771:                        refFrames, refValues, resolution)) {
Chris@771:         return false;
Chris@771:     }
Chris@771: 
Chris@771:     if (!getValuesFrom(m_toAlignOutputModel,
Chris@771:                        otherFrames, otherValues, resolution)) {
Chris@771:         return false;
Chris@771:     }
Chris@767:     
Chris@767:     vector<double> s1, s2;
Chris@771:     for (double v: refValues) {
Chris@771:         s1.push_back(m_magnitudePreprocessor(v));
Chris@771:     }
Chris@771:     for (double v: otherValues) {
Chris@771:         s2.push_back(m_magnitudePreprocessor(v));
Chris@767:     }
Chris@767: 
Chris@769:     SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentMagnitude: "
Chris@767:            << "Have " << s1.size() << " events from reference, "
Chris@767:            << s2.size() << " from toAlign" << endl;
Chris@771:     
Chris@767:     MagnitudeDTW dtw;
Chris@767:     vector<size_t> alignment;
Chris@767: 
Chris@767:     {
Chris@767:         SVCERR << "TransformDTWAligner[" << this
Chris@767:                << "]: serialising DTW to avoid over-allocation" << endl;
Chris@771:         QMutexLocker locker(&m_dtwMutex);
Chris@767:         alignment = dtw.alignSeries(s1, s2);
Chris@767:     }
Chris@767: 
Chris@769:     SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentMagnitude: "
Chris@767:            << "DTW produced " << alignment.size() << " points:" << endl;
Chris@771:     for (int i = 0; in_range_for(alignment, i) && i < 100; ++i) {
Chris@767:         SVCERR << alignment[i] << " ";
Chris@767:     }
Chris@767:     SVCERR << endl;
Chris@767: 
Chris@771:     alignmentModel->setPath(makePath(alignment,
Chris@771:                                      refFrames,
Chris@771:                                      otherFrames,
Chris@771:                                      alignmentModel->getSampleRate(),
Chris@771:                                      resolution));
Chris@768:     alignmentModel->setCompletion(100);
Chris@767: 
Chris@771:     SVCERR << "TransformDTWAligner[" << this
Chris@771:            << "]: performAlignmentMagnitude: Done" << endl;
Chris@767: 
Chris@767:     m_incomplete = false;
Chris@767:     return true;
Chris@767: }
Chris@767: 
Chris@767: bool
Chris@767: TransformDTWAligner::performAlignmentRiseFall()
Chris@767: {
Chris@771:     auto alignmentModel = ModelById::getAs<AlignmentModel>(m_alignmentModel);
Chris@767:     if (!alignmentModel) {
Chris@767:         return false;
Chris@767:     }
Chris@768: 
Chris@771:     vector<sv_frame_t> refFrames, otherFrames;
Chris@771:     vector<double> refValues, otherValues;
Chris@771:     sv_frame_t resolution = 0;
Chris@771: 
Chris@771:     if (!getValuesFrom(m_referenceOutputModel,
Chris@771:                        refFrames, refValues, resolution)) {
Chris@771:         return false;
Chris@771:     }
Chris@771: 
Chris@771:     if (!getValuesFrom(m_toAlignOutputModel,
Chris@771:                        otherFrames, otherValues, resolution)) {
Chris@771:         return false;
Chris@771:     }
Chris@771:     
Chris@771:     auto preprocess =
Chris@771:         [this](const std::vector<double> &vv) {
Chris@768:             vector<RiseFallDTW::Value> s;
Chris@768:             double prev = 0.0;
Chris@771:             for (auto curr: vv) {
Chris@771:                 s.push_back(m_riseFallPreprocessor(prev, curr));
Chris@771:                 prev = curr;
Chris@768:             }
Chris@768:             return s;
Chris@771:         }; 
Chris@767:     
Chris@771:     vector<RiseFallDTW::Value> s1 = preprocess(refValues);
Chris@771:     vector<RiseFallDTW::Value> s2 = preprocess(otherValues);
Chris@767: 
Chris@769:     SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentRiseFall: "
Chris@767:            << "Have " << s1.size() << " events from reference, "
Chris@767:            << s2.size() << " from toAlign" << endl;
Chris@767: 
Chris@771:     SVCERR << "Reference:" << endl;
Chris@771:     for (int i = 0; in_range_for(s1, i) && i < 100; ++i) {
Chris@771:         SVCERR << s1[i] << " ";
Chris@771:     }
Chris@771:     SVCERR << endl;
Chris@771: 
Chris@771:     SVCERR << "toAlign:" << endl;
Chris@771:     for (int i = 0; in_range_for(s2, i) && i < 100; ++i) {
Chris@771:         SVCERR << s2[i] << " ";
Chris@771:     }
Chris@771:     SVCERR << endl;
Chris@771: 
Chris@767:     RiseFallDTW dtw;
Chris@767:     vector<size_t> alignment;
Chris@767: 
Chris@767:     {
Chris@767:         SVCERR << "TransformDTWAligner[" << this
Chris@767:                << "]: serialising DTW to avoid over-allocation" << endl;
Chris@771:         QMutexLocker locker(&m_dtwMutex);
Chris@767:         alignment = dtw.alignSeries(s1, s2);
Chris@767:     }
Chris@767: 
Chris@769:     SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentRiseFall: "
Chris@767:            << "DTW produced " << alignment.size() << " points:" << endl;
Chris@767:     for (int i = 0; i < alignment.size() && i < 100; ++i) {
Chris@767:         SVCERR << alignment[i] << " ";
Chris@767:     }
Chris@767:     SVCERR << endl;
Chris@767: 
Chris@771:     alignmentModel->setPath(makePath(alignment,
Chris@771:                                      refFrames,
Chris@771:                                      otherFrames,
Chris@771:                                      alignmentModel->getSampleRate(),
Chris@771:                                      resolution));
Chris@771: 
Chris@768:     alignmentModel->setCompletion(100);
Chris@767: 
Chris@769:     SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentRiseFall: Done"
Chris@767:            << endl;
Chris@767: 
Chris@767:     m_incomplete = false;
Chris@767:     return true;
Chris@767: }