Chris@767: /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ Chris@767: Chris@767: /* Chris@767: Sonic Visualiser Chris@767: An audio file viewer and annotation editor. Chris@767: Centre for Digital Music, Queen Mary, University of London. Chris@767: Chris@767: This program is free software; you can redistribute it and/or Chris@767: modify it under the terms of the GNU General Public License as Chris@767: published by the Free Software Foundation; either version 2 of the Chris@767: License, or (at your option) any later version. See the file Chris@767: COPYING included with this distribution for more information. Chris@767: */ Chris@767: Chris@767: #include "TransformDTWAligner.h" Chris@767: #include "DTW.h" Chris@767: Chris@767: #include "data/model/SparseTimeValueModel.h" Chris@767: #include "data/model/RangeSummarisableTimeValueModel.h" Chris@767: #include "data/model/AlignmentModel.h" Chris@767: #include "data/model/AggregateWaveModel.h" Chris@767: Chris@767: #include "framework/Document.h" Chris@767: Chris@767: #include "transform/ModelTransformerFactory.h" Chris@767: #include "transform/FeatureExtractionModelTransformer.h" Chris@767: Chris@767: #include Chris@767: #include Chris@767: #include Chris@767: Chris@767: using std::vector; Chris@767: Chris@767: TransformDTWAligner::TransformDTWAligner(Document *doc, Chris@767: ModelId reference, Chris@767: ModelId toAlign, Chris@767: Transform transform, Chris@767: DTWType dtwType) : Chris@767: m_document(doc), Chris@767: m_reference(reference), Chris@767: m_toAlign(toAlign), Chris@767: m_transform(transform), Chris@767: m_dtwType(dtwType), Chris@768: m_incomplete(true), Chris@768: m_outputPreprocessor([](double x) { return x; }) Chris@768: { Chris@768: } Chris@768: Chris@768: TransformDTWAligner::TransformDTWAligner(Document *doc, Chris@768: ModelId reference, Chris@768: ModelId toAlign, Chris@768: Transform transform, Chris@768: DTWType dtwType, Chris@768: std::function Chris@768: outputPreprocessor) : Chris@768: m_document(doc), Chris@768: m_reference(reference), Chris@768: m_toAlign(toAlign), Chris@768: m_transform(transform), Chris@768: m_dtwType(dtwType), Chris@768: m_incomplete(true), Chris@768: m_outputPreprocessor(outputPreprocessor) Chris@767: { Chris@767: } Chris@767: Chris@767: TransformDTWAligner::~TransformDTWAligner() Chris@767: { Chris@767: if (m_incomplete) { Chris@767: if (auto toAlign = ModelById::get(m_toAlign)) { Chris@767: toAlign->setAlignment({}); Chris@767: } Chris@767: } Chris@767: Chris@767: ModelById::release(m_referenceOutputModel); Chris@767: ModelById::release(m_toAlignOutputModel); Chris@767: } Chris@767: Chris@767: bool Chris@767: TransformDTWAligner::isAvailable() Chris@767: { Chris@767: //!!! needs to be isAvailable(QString transformId)? Chris@767: return true; Chris@767: } Chris@767: Chris@767: void Chris@767: TransformDTWAligner::begin() Chris@767: { Chris@767: auto reference = Chris@767: ModelById::getAs(m_reference); Chris@767: auto toAlign = Chris@767: ModelById::getAs(m_toAlign); Chris@767: Chris@767: if (!reference || !toAlign) return; Chris@767: Chris@767: SVCERR << "TransformDTWAligner[" << this << "]: begin(): aligning " Chris@767: << m_toAlign << " against reference " << m_reference << endl; Chris@767: Chris@767: ModelTransformerFactory *mtf = ModelTransformerFactory::getInstance(); Chris@767: Chris@767: QString message; Chris@767: Chris@767: m_referenceOutputModel = mtf->transform(m_transform, m_reference, message); Chris@767: auto referenceOutputModel = ModelById::get(m_referenceOutputModel); Chris@767: if (!referenceOutputModel) { Chris@767: SVCERR << "Align::alignModel: ERROR: Failed to create reference output model (no plugin?)" << endl; Chris@767: emit failed(m_toAlign, message); Chris@767: return; Chris@767: } Chris@767: Chris@767: SVCERR << "TransformDTWAligner[" << this << "]: begin(): transform id " Chris@767: << m_transform.getIdentifier() Chris@767: << " is running on reference model" << endl; Chris@767: Chris@767: message = ""; Chris@767: Chris@767: m_toAlignOutputModel = mtf->transform(m_transform, m_toAlign, message); Chris@767: auto toAlignOutputModel = ModelById::get(m_toAlignOutputModel); Chris@767: if (!toAlignOutputModel) { Chris@767: SVCERR << "Align::alignModel: ERROR: Failed to create toAlign output model (no plugin?)" << endl; Chris@767: emit failed(m_toAlign, message); Chris@767: return; Chris@767: } Chris@767: Chris@767: SVCERR << "TransformDTWAligner[" << this << "]: begin(): transform id " Chris@767: << m_transform.getIdentifier() Chris@767: << " is running on toAlign model" << endl; Chris@767: Chris@767: connect(referenceOutputModel.get(), SIGNAL(completionChanged(ModelId)), Chris@767: this, SLOT(completionChanged(ModelId))); Chris@767: connect(toAlignOutputModel.get(), SIGNAL(completionChanged(ModelId)), Chris@767: this, SLOT(completionChanged(ModelId))); Chris@767: Chris@767: auto alignmentModel = std::make_shared Chris@768: (m_reference, m_toAlign, ModelId()); Chris@767: m_alignmentModel = ModelById::add(alignmentModel); Chris@767: Chris@767: toAlign->setAlignment(m_alignmentModel); Chris@767: m_document->addNonDerivedModel(m_alignmentModel); Chris@767: Chris@767: // we wouldn't normally expect these to be true here, but... Chris@767: int completion = 0; Chris@767: if (referenceOutputModel->isReady(&completion) && Chris@767: toAlignOutputModel->isReady(&completion)) { Chris@767: SVCERR << "TransformDTWAligner[" << this << "]: begin(): output models " Chris@767: << "are ready already! calling performAlignment" << endl; Chris@767: if (performAlignment()) { Chris@767: emit complete(m_alignmentModel); Chris@767: } else { Chris@767: emit failed(m_toAlign, tr("Failed to calculate alignment using DTW")); Chris@767: } Chris@767: } Chris@767: } Chris@767: Chris@767: void Chris@767: TransformDTWAligner::completionChanged(ModelId id) Chris@767: { Chris@767: if (!m_incomplete) { Chris@767: return; Chris@767: } Chris@767: Chris@767: SVCERR << "TransformDTWAligner[" << this << "]: completionChanged: " Chris@767: << "model " << id << endl; Chris@767: Chris@767: auto referenceOutputModel = ModelById::get(m_referenceOutputModel); Chris@767: auto toAlignOutputModel = ModelById::get(m_toAlignOutputModel); Chris@768: auto alignmentModel = ModelById::getAs(m_alignmentModel); Chris@767: Chris@768: if (!referenceOutputModel || !toAlignOutputModel || !alignmentModel) { Chris@767: return; Chris@767: } Chris@767: Chris@767: int referenceCompletion = 0, toAlignCompletion = 0; Chris@767: bool referenceReady = referenceOutputModel->isReady(&referenceCompletion); Chris@767: bool toAlignReady = toAlignOutputModel->isReady(&toAlignCompletion); Chris@767: Chris@767: if (referenceReady && toAlignReady) { Chris@767: Chris@767: SVCERR << "TransformDTWAligner[" << this << "]: completionChanged: " Chris@767: << "ready, calling performAlignment" << endl; Chris@767: Chris@768: alignmentModel->setCompletion(95); Chris@767: Chris@767: if (performAlignment()) { Chris@767: emit complete(m_alignmentModel); Chris@767: } else { Chris@767: emit failed(m_toAlign, tr("Alignment of transform outputs failed")); Chris@767: } Chris@767: Chris@767: } else { Chris@767: Chris@767: SVCERR << "TransformDTWAligner[" << this << "]: completionChanged: " Chris@767: << "not ready yet: reference completion " << referenceCompletion Chris@767: << ", toAlign completion " << toAlignCompletion << endl; Chris@767: Chris@768: int completion = std::min(referenceCompletion, Chris@768: toAlignCompletion); Chris@768: completion = (completion * 94) / 100; Chris@768: alignmentModel->setCompletion(completion); Chris@767: } Chris@767: } Chris@767: Chris@767: bool Chris@767: TransformDTWAligner::performAlignment() Chris@767: { Chris@767: if (m_dtwType == Magnitude) { Chris@767: return performAlignmentMagnitude(); Chris@767: } else { Chris@767: return performAlignmentRiseFall(); Chris@767: } Chris@767: } Chris@767: Chris@767: bool Chris@767: TransformDTWAligner::performAlignmentMagnitude() Chris@767: { Chris@767: auto referenceOutputSTVM = ModelById::getAs Chris@767: (m_referenceOutputModel); Chris@767: auto toAlignOutputSTVM = ModelById::getAs Chris@767: (m_toAlignOutputModel); Chris@767: auto alignmentModel = ModelById::getAs Chris@767: (m_alignmentModel); Chris@767: Chris@767: if (!referenceOutputSTVM || !toAlignOutputSTVM) { Chris@767: //!!! what? Chris@767: return false; Chris@767: } Chris@767: Chris@767: if (!alignmentModel) { Chris@767: return false; Chris@767: } Chris@767: Chris@767: vector s1, s2; Chris@767: Chris@767: { Chris@767: auto events = referenceOutputSTVM->getAllEvents(); Chris@767: for (auto e: events) { Chris@768: s1.push_back(m_outputPreprocessor(e.getValue())); Chris@767: } Chris@767: events = toAlignOutputSTVM->getAllEvents(); Chris@767: for (auto e: events) { Chris@768: s2.push_back(m_outputPreprocessor(e.getValue())); Chris@767: } Chris@767: } Chris@767: Chris@769: SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentMagnitude: " Chris@767: << "Have " << s1.size() << " events from reference, " Chris@767: << s2.size() << " from toAlign" << endl; Chris@767: Chris@767: MagnitudeDTW dtw; Chris@767: vector alignment; Chris@767: Chris@767: { Chris@767: SVCERR << "TransformDTWAligner[" << this Chris@767: << "]: serialising DTW to avoid over-allocation" << endl; Chris@767: static QMutex mutex; Chris@767: QMutexLocker locker(&mutex); Chris@767: Chris@767: alignment = dtw.alignSeries(s1, s2); Chris@767: } Chris@767: Chris@769: SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentMagnitude: " Chris@767: << "DTW produced " << alignment.size() << " points:" << endl; Chris@767: for (int i = 0; i < alignment.size() && i < 100; ++i) { Chris@767: SVCERR << alignment[i] << " "; Chris@767: } Chris@767: SVCERR << endl; Chris@767: Chris@768: alignmentModel->setCompletion(100); Chris@767: Chris@767: sv_frame_t resolution = referenceOutputSTVM->getResolution(); Chris@767: sv_frame_t sourceFrame = 0; Chris@767: Chris@767: Path path(referenceOutputSTVM->getSampleRate(), resolution); Chris@767: Chris@767: for (size_t m: alignment) { Chris@767: path.add(PathPoint(sourceFrame, sv_frame_t(m) * resolution)); Chris@767: sourceFrame += resolution; Chris@767: } Chris@767: Chris@767: alignmentModel->setPath(path); Chris@767: Chris@769: SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentMagnitude: Done" Chris@767: << endl; Chris@767: Chris@767: m_incomplete = false; Chris@767: return true; Chris@767: } Chris@767: Chris@767: bool Chris@767: TransformDTWAligner::performAlignmentRiseFall() Chris@767: { Chris@767: auto referenceOutputSTVM = ModelById::getAs Chris@767: (m_referenceOutputModel); Chris@767: auto toAlignOutputSTVM = ModelById::getAs Chris@767: (m_toAlignOutputModel); Chris@767: auto alignmentModel = ModelById::getAs Chris@767: (m_alignmentModel); Chris@767: Chris@767: if (!referenceOutputSTVM || !toAlignOutputSTVM) { Chris@767: //!!! what? Chris@767: return false; Chris@767: } Chris@767: Chris@767: if (!alignmentModel) { Chris@767: return false; Chris@767: } Chris@768: Chris@768: auto convertEvents = Chris@768: [this](const EventVector &ee) { Chris@768: vector s; Chris@768: double prev = 0.0; Chris@768: for (auto e: ee) { Chris@768: double v = m_outputPreprocessor(e.getValue()); Chris@768: if (v == prev || s.empty()) { Chris@768: s.push_back({ RiseFallDTW::Direction::None, 0.0 }); Chris@768: } else if (v > prev) { Chris@768: s.push_back({ RiseFallDTW::Direction::Up, v - prev }); Chris@768: } else { Chris@768: s.push_back({ RiseFallDTW::Direction::Down, prev - v }); Chris@768: } Chris@768: } Chris@768: return s; Chris@768: }; Chris@767: Chris@768: vector s1 = Chris@768: convertEvents(referenceOutputSTVM->getAllEvents()); Chris@767: Chris@768: vector s2 = Chris@768: convertEvents(toAlignOutputSTVM->getAllEvents()); Chris@767: Chris@769: SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentRiseFall: " Chris@767: << "Have " << s1.size() << " events from reference, " Chris@767: << s2.size() << " from toAlign" << endl; Chris@767: Chris@767: RiseFallDTW dtw; Chris@767: Chris@767: vector alignment; Chris@767: Chris@767: { Chris@767: SVCERR << "TransformDTWAligner[" << this Chris@767: << "]: serialising DTW to avoid over-allocation" << endl; Chris@767: static QMutex mutex; Chris@767: QMutexLocker locker(&mutex); Chris@767: Chris@767: alignment = dtw.alignSeries(s1, s2); Chris@767: } Chris@767: Chris@769: SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentRiseFall: " Chris@767: << "DTW produced " << alignment.size() << " points:" << endl; Chris@767: for (int i = 0; i < alignment.size() && i < 100; ++i) { Chris@767: SVCERR << alignment[i] << " "; Chris@767: } Chris@767: SVCERR << endl; Chris@767: Chris@768: alignmentModel->setCompletion(100); Chris@767: Chris@767: sv_frame_t resolution = referenceOutputSTVM->getResolution(); Chris@767: sv_frame_t sourceFrame = 0; Chris@767: Chris@767: Path path(referenceOutputSTVM->getSampleRate(), resolution); Chris@767: Chris@767: for (size_t m: alignment) { Chris@767: path.add(PathPoint(sourceFrame, sv_frame_t(m) * resolution)); Chris@767: sourceFrame += resolution; Chris@767: } Chris@767: Chris@767: alignmentModel->setPath(path); Chris@767: Chris@769: SVCERR << "TransformDTWAligner[" << this << "]: performAlignmentRiseFall: Done" Chris@767: << endl; Chris@767: Chris@767: m_incomplete = false; Chris@767: return true; Chris@767: }