Mercurial > hg > svapp
changeset 763:da57ab54f0e8
Merge from branch pitch-align. Doesn't actually do pitch alignment here, but this is the groundwork.
author | Chris Cannam |
---|---|
date | Wed, 13 May 2020 14:10:47 +0100 |
parents | 3a63f1f61bd6 (current diff) 6429a164b7e1 (diff) |
children | 4c91c95e146a |
files | framework/MainWindowBase.cpp framework/MainWindowBase.h |
diffstat | 15 files changed, 1269 insertions(+), 819 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/Align.cpp Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,143 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "Align.h" +#include "TransformAligner.h" +#include "ExternalProgramAligner.h" +#include "framework/Document.h" + +#include <QSettings> +#include <QTimer> + +void +Align::alignModel(Document *doc, + ModelId reference, + ModelId toAlign) +{ + addAligner(doc, reference, toAlign); + m_aligners[toAlign]->begin(); +} + +void +Align::scheduleAlignment(Document *doc, + ModelId reference, + ModelId toAlign) +{ + addAligner(doc, reference, toAlign); + int delay = 500 + 500 * int(m_aligners.size()); + if (delay > 3500) { + delay = 3500; + } + SVCERR << "Align::scheduleAlignment: delaying " << delay << "ms" << endl; + QTimer::singleShot(delay, m_aligners[toAlign].get(), SLOT(begin())); +} + +void +Align::addAligner(Document *doc, + ModelId reference, + ModelId toAlign) +{ + bool useProgram; + QString program; + getAlignerPreference(useProgram, program); + + std::shared_ptr<Aligner> aligner; + + { + // Replace the aligner with a new one. This also stops any + // previously-running alignment, when the old entry is + // replaced and its aligner destroyed. + + QMutexLocker locker(&m_mutex); + + if (useProgram && (program != "")) { + m_aligners[toAlign] = + std::make_shared<ExternalProgramAligner>(doc, + reference, + toAlign, + program); + } else { + m_aligners[toAlign] = + std::make_shared<TransformAligner>(doc, + reference, + toAlign); + } + + aligner = m_aligners[toAlign]; + } + + connect(aligner.get(), SIGNAL(complete(ModelId)), + this, SLOT(alignerComplete(ModelId))); + + connect(aligner.get(), SIGNAL(failed(ModelId, QString)), + this, SLOT(alignerFailed(ModelId, QString))); +} + +void +Align::getAlignerPreference(bool &useProgram, QString &program) +{ + QSettings settings; + settings.beginGroup("Preferences"); + useProgram = settings.value("use-external-alignment", false).toBool(); + program = settings.value("external-alignment-program", "").toString(); + settings.endGroup(); +} + +bool +Align::canAlign() +{ + bool useProgram; + QString program; + getAlignerPreference(useProgram, program); + + if (useProgram) { + return ExternalProgramAligner::isAvailable(program); + } else { + return TransformAligner::isAvailable(); + } +} + +void +Align::alignerComplete(ModelId alignmentModel) +{ + removeAligner(sender()); + emit alignmentComplete(alignmentModel); +} + +void +Align::alignerFailed(ModelId toAlign, QString error) +{ + removeAligner(sender()); + emit alignmentFailed(toAlign, error); +} + +void +Align::removeAligner(QObject *obj) +{ + Aligner *aligner = qobject_cast<Aligner *>(obj); + if (!aligner) { + SVCERR << "ERROR: Align::removeAligner: Not an Aligner" << endl; + return; + } + + QMutexLocker locker (&m_mutex); + + for (auto p: m_aligners) { + if (aligner == p.second.get()) { + m_aligners.erase(p.first); + break; + } + } +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/Align.h Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,121 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef SV_ALIGN_H +#define SV_ALIGN_H + +#include <QString> +#include <QObject> +#include <QProcess> +#include <QMutex> +#include <set> + +#include "Aligner.h" + +class AlignmentModel; +class Document; + +class Align : public QObject +{ + Q_OBJECT + +public: + Align() { } + + /** + * Align the "other" model to the reference, attaching an + * AlignmentModel to it. Alignment is carried out by the method + * configured in the user preferences (either a plugin transform + * or an external process) and is done asynchronously. + * + * Any errors are reported by firing the alignmentFailed + * signal. Note that the signal may be fired during the call to + * this function, if the aligner fails to start at all. + * + * If alignment starts successfully, then an AlignmentModel has + * been constructed and attached to the toAlign model, and you can + * query that model to discover the alignment progress, eventual + * outcome, and also (separately from the alignmentFailed signal + * here) any error message generated during alignment. + * + * A single Align object may carry out many simultanous alignment + * calls -- you do not need to create a new Align object each + * time, nor to wait for an alignment to be complete before + * starting a new one. + * + * The Align object must survive after this call, for at least as + * long as the alignment takes. The usual expectation is that the + * Align object will simply share the process or document + * lifespan. + */ + void alignModel(Document *doc, + ModelId reference, + ModelId toAlign); + + /** + * As alignModel, except that the alignment does not begin + * immediately, but is instead placed behind an event callback + * with a small delay. Useful to avoid an unresponsive GUI when + * firing off alignments while doing something else as well. Any + * error is reported by firing the alignmentFailed signal. + * + * Scheduled alignments are not queued or serialised - many could + * happen at once. They are just delayed a little for UI + * responsiveness. + */ + void scheduleAlignment(Document *doc, + ModelId reference, + ModelId toAlign); + + /** + * Return true if the alignment facility is available (relevant + * plugin installed, etc). + */ + static bool canAlign(); + +signals: + /** + * Emitted when an alignment is successfully completed. The + * reference and other models can be queried from the alignment + * model. + */ + void alignmentComplete(ModelId alignmentModel); // an AlignmentModel + + /** + * Emitted when an alignment fails. The model is the toAlign model + * that was passed to the call to alignModel or scheduleAlignment. + */ + void alignmentFailed(ModelId toAlign, QString errorText); + +private slots: + void alignerComplete(ModelId alignmentModel); // an AlignmentModel + void alignerFailed(ModelId toAlign, QString errorText); + +private: + QMutex m_mutex; + + // maps toAlign -> aligner for ongoing alignment - note that + // although we can calculate alignments with different references, + // we can only have one alignment on any given toAlign model, so + // we don't key this on the whole (reference, toAlign) pair + std::map<ModelId, std::shared_ptr<Aligner>> m_aligners; + + void addAligner(Document *doc, ModelId reference, ModelId toAlign); + void removeAligner(QObject *); + + static void getAlignerPreference(bool &useProgram, QString &program); +}; + +#endif +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/Aligner.h Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,45 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef SV_ALIGNER_H +#define SV_ALIGNER_H + +#include <QString> + +#include "data/model/Model.h" + +class Aligner : public QObject +{ + Q_OBJECT + +public: + virtual ~Aligner() { } + +public slots: + virtual void begin() = 0; + +signals: + /** + * Emitted when alignment is successfully completed. The reference + * and toAlign models can be queried from the alignment model. + */ + void complete(ModelId alignmentModel); // an AlignmentModel + + /** + * Emitted when alignment fails. + */ + void failed(ModelId toAlign, QString errorText); // the toAlign model +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/DTW.h Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,192 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef SV_DTW_H +#define SV_DTW_H + +#include <vector> +#include <functional> + +template <typename Value> +class DTW +{ +public: + DTW(std::function<double(const Value &, const Value &)> distanceMetric) : + m_metric(distanceMetric) { } + + std::vector<size_t> alignSeries(std::vector<Value> s1, + std::vector<Value> s2) { + + // Return the index into s2 for each element in s1 + + std::vector<size_t> alignment(s1.size(), 0); + + if (s1.empty() || s2.empty()) { + return alignment; + } + + auto costs = costSeries(s1, s2); + + size_t j = s1.size() - 1; + size_t i = s2.size() - 1; + + while (j > 0 && i > 0) { + + alignment[j] = i; + + cost_t a = costs[j-1][i]; + cost_t b = costs[j][i-1]; + cost_t both = costs[j-1][i-1]; + + if (a < b) { + --j; + if (both < a) { + --i; + } + } else { + --i; + if (both < b) { + --j; + } + } + } + + return alignment; + } + +private: + std::function<double(const Value &, const Value &)> m_metric; + + typedef double cost_t; + + struct CostOption { + bool present; + cost_t cost; + }; + + cost_t choose(CostOption x, CostOption y, CostOption d) { + if (x.present && y.present) { + if (!d.present) { + throw std::logic_error("if x & y both exist, so must diagonal"); + } + return std::min(std::min(x.cost, y.cost), d.cost); + } else if (x.present) { + return x.cost; + } else if (y.present) { + return y.cost; + } else { + return 0.0; + } + } + + std::vector<std::vector<cost_t>> costSeries(std::vector<Value> s1, + std::vector<Value> s2) { + + std::vector<std::vector<cost_t>> costs + (s1.size(), std::vector<cost_t>(s2.size(), 0.0)); + + for (size_t j = 0; j < s1.size(); ++j) { + for (size_t i = 0; i < s2.size(); ++i) { + cost_t c = m_metric(s1[j], s2[i]); + costs[j][i] = choose + ( + { j > 0, + j > 0 ? c + costs[j-1][i] : 0.0 + }, + { i > 0, + i > 0 ? c + costs[j][i-1] : 0.0 + }, + { j > 0 && i > 0, + j > 0 && i > 0 ? c + costs[j-1][i-1] : 0.0 + }); + } + } + + return costs; + } +}; + +class MagnitudeDTW +{ +public: + MagnitudeDTW() : m_dtw(metric) { } + + std::vector<size_t> alignSeries(std::vector<double> s1, + std::vector<double> s2) { + return m_dtw.alignSeries(s1, s2); + } + +private: + DTW<double> m_dtw; + + static double metric(const double &a, const double &b) { + return std::abs(b - a); + } +}; + +class RiseFallDTW +{ +public: + enum class Direction { + None, + Up, + Down + }; + + struct Value { + Direction direction; + double distance; + }; + + RiseFallDTW() : m_dtw(metric) { } + + std::vector<size_t> alignSeries(std::vector<Value> s1, + std::vector<Value> s2) { + return m_dtw.alignSeries(s1, s2); + } + +private: + DTW<Value> m_dtw; + + static double metric(const Value &a, const Value &b) { + + auto together = [](double c1, double c2) { + auto diff = std::abs(c1 - c2); + return (diff < 1.0 ? -1.0 : + diff > 3.0 ? 1.0 : + 0.0); + }; + auto opposing = [](double c1, double c2) { + auto diff = c1 + c2; + return (diff < 2.0 ? 1.0 : + 2.0); + }; + + if (a.direction == Direction::None || b.direction == Direction::None) { + if (a.direction == b.direction) { + return 0.0; + } else { + return 1.0; + } + } else { + if (a.direction == b.direction) { + return together (a.distance, b.distance); + } else { + return opposing (a.distance, b.distance); + } + } + } +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/ExternalProgramAligner.cpp Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,231 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "ExternalProgramAligner.h" + +#include <QFileInfo> +#include <QApplication> + +#include "data/model/ReadOnlyWaveFileModel.h" +#include "data/model/SparseTimeValueModel.h" +#include "data/model/AlignmentModel.h" + +#include "data/fileio/CSVFileReader.h" + +#include "framework/Document.h" + +ExternalProgramAligner::ExternalProgramAligner(Document *doc, + ModelId reference, + ModelId toAlign, + QString program) : + m_document(doc), + m_reference(reference), + m_toAlign(toAlign), + m_program(program), + m_process(nullptr) +{ +} + +ExternalProgramAligner::~ExternalProgramAligner() +{ + delete m_process; +} + +bool +ExternalProgramAligner::isAvailable(QString program) +{ + QFileInfo file(program); + return file.exists() && file.isExecutable(); +} + +void +ExternalProgramAligner::begin() +{ + // Run an external program, passing to it paths to the main + // model's audio file and the new model's audio file. It returns + // the path in CSV form through stdout. + + auto reference = ModelById::getAs<ReadOnlyWaveFileModel>(m_reference); + auto other = ModelById::getAs<ReadOnlyWaveFileModel>(m_toAlign); + if (!reference || !other) { + SVCERR << "ERROR: ExternalProgramAligner: Can't align non-read-only models via program (no local filename available)" << endl; + return; + } + + while (!reference->isReady(nullptr) || !other->isReady(nullptr)) { + qApp->processEvents(); + } + + QString refPath = reference->getLocalFilename(); + if (refPath == "") { + refPath = FileSource(reference->getLocation()).getLocalFilename(); + } + + QString otherPath = other->getLocalFilename(); + if (otherPath == "") { + otherPath = FileSource(other->getLocation()).getLocalFilename(); + } + + if (refPath == "" || otherPath == "") { + emit failed(m_toAlign, + tr("Failed to find local filepath for wave-file model")); + return; + } + + auto alignmentModel = + std::make_shared<AlignmentModel>(m_reference, m_toAlign, ModelId()); + + m_alignmentModel = ModelById::add(alignmentModel); + other->setAlignment(m_alignmentModel); + + m_process = new QProcess; + m_process->setProcessChannelMode(QProcess::ForwardedErrorChannel); + + connect(m_process, + SIGNAL(finished(int, QProcess::ExitStatus)), + this, + SLOT(programFinished(int, QProcess::ExitStatus))); + + QStringList args; + args << refPath << otherPath; + + SVCERR << "ExternalProgramAligner: Starting program \"" + << m_program << "\" with args: "; + for (auto a: args) { + SVCERR << "\"" << a << "\" "; + } + SVCERR << endl; + + m_process->start(m_program, args); + + bool success = m_process->waitForStarted(); + + if (!success) { + + SVCERR << "ERROR: ExternalProgramAligner: Program did not start" << endl; + emit failed(m_toAlign, + tr("Alignment program \"%1\" did not start") + .arg(m_program)); + + other->setAlignment({}); + ModelById::release(m_alignmentModel); + delete m_process; + m_process = nullptr; + + } else { + m_document->addNonDerivedModel(m_alignmentModel); + } +} + +void +ExternalProgramAligner::programFinished(int exitCode, QProcess::ExitStatus status) +{ + SVCERR << "ExternalProgramAligner::programFinished" << endl; + + QProcess *process = qobject_cast<QProcess *>(sender()); + + if (process != m_process) { + SVCERR << "ERROR: ExternalProgramAligner: Emitting process " << process + << " is not my process!" << endl; + return; + } + + auto alignmentModel = ModelById::getAs<AlignmentModel>(m_alignmentModel); + if (!alignmentModel) { + SVCERR << "ExternalProgramAligner: AlignmentModel no longer exists" + << endl; + return; + } + + if (exitCode == 0 && status == 0) { + + CSVFormat format; + format.setModelType(CSVFormat::TwoDimensionalModel); + format.setTimingType(CSVFormat::ExplicitTiming); + format.setTimeUnits(CSVFormat::TimeSeconds); + format.setColumnCount(2); + // The output format has time in the reference file first, and + // time in the "other" file in the second column. This is a + // more natural approach for a command-line alignment tool, + // but it's the opposite of what we expect for native + // alignment paths, which map from "other" file to + // reference. These column purpose settings reflect that. + format.setColumnPurpose(1, CSVFormat::ColumnStartTime); + format.setColumnPurpose(0, CSVFormat::ColumnValue); + format.setAllowQuoting(false); + format.setSeparator(','); + + CSVFileReader reader(process, format, alignmentModel->getSampleRate()); + if (!reader.isOK()) { + SVCERR << "ERROR: ExternalProgramAligner: Failed to parse output" + << endl; + QString error = tr("Failed to parse output of program: %1") + .arg(reader.getError()); + alignmentModel->setError(error); + emit failed(m_toAlign, error); + goto done; + } + + //!!! to use ById? + + Model *csvOutput = reader.load(); + + SparseTimeValueModel *path = + qobject_cast<SparseTimeValueModel *>(csvOutput); + if (!path) { + SVCERR << "ERROR: ExternalProgramAligner: Output did not convert to sparse time-value model" + << endl; + QString error = + tr("Output of alignment program was not in the proper format"); + alignmentModel->setError(error); + delete csvOutput; + emit failed(m_toAlign, error); + goto done; + } + + if (path->isEmpty()) { + SVCERR << "ERROR: ExternalProgramAligner: Output contained no mappings" + << endl; + QString error = + tr("Output of alignment program contained no mappings"); + alignmentModel->setError(error); + delete path; + emit failed(m_toAlign, error); + goto done; + } + + SVCERR << "ExternalProgramAligner: Setting alignment path (" + << path->getEventCount() << " point(s))" << endl; + + auto pathId = + ModelById::add(std::shared_ptr<SparseTimeValueModel>(path)); + alignmentModel->setPathFrom(pathId); + + emit complete(m_alignmentModel); + + ModelById::release(pathId); + + } else { + SVCERR << "ERROR: ExternalProgramAligner: Aligner program " + << "failed: exit code " << exitCode << ", status " << status + << endl; + QString error = tr("Aligner process returned non-zero exit status"); + alignmentModel->setError(error); + emit failed(m_toAlign, error); + } + +done: + delete m_process; + m_process = nullptr; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/ExternalProgramAligner.h Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,55 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef SV_EXTERNAL_PROGRAM_ALIGNER_H +#define SV_EXTERNAL_PROGRAM_ALIGNER_H + +#include "Aligner.h" + +#include <QProcess> +#include <QString> + +class AlignmentModel; +class Document; + +class ExternalProgramAligner : public Aligner +{ + Q_OBJECT + +public: + ExternalProgramAligner(Document *doc, + ModelId reference, + ModelId toAlign, + QString program); + + // Destroy the aligner, cleanly cancelling any ongoing alignment + ~ExternalProgramAligner(); + + void begin() override; + + static bool isAvailable(QString program); + +private slots: + void programFinished(int, QProcess::ExitStatus); + +private: + Document *m_document; + ModelId m_reference; + ModelId m_toAlign; + ModelId m_alignmentModel; + QString m_program; + QProcess *m_process; +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/TransformAligner.cpp Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,404 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "TransformAligner.h" + +#include "data/model/SparseTimeValueModel.h" +#include "data/model/RangeSummarisableTimeValueModel.h" +#include "data/model/AlignmentModel.h" +#include "data/model/AggregateWaveModel.h" + +#include "framework/Document.h" + +#include "transform/TransformFactory.h" +#include "transform/ModelTransformerFactory.h" +#include "transform/FeatureExtractionModelTransformer.h" + +#include <QSettings> + +TransformAligner::TransformAligner(Document *doc, + ModelId reference, + ModelId toAlign) : + m_document(doc), + m_reference(reference), + m_toAlign(toAlign), + m_tuningFrequency(440.f), + m_incomplete(true) +{ +} + +TransformAligner::~TransformAligner() +{ + if (m_incomplete) { + auto other = + ModelById::getAs<RangeSummarisableTimeValueModel>(m_toAlign); + if (other) { + other->setAlignment({}); + } + } + + ModelById::release(m_tuningDiffProgressModel); + ModelById::release(m_tuningDiffOutputModel); + ModelById::release(m_pathOutputModel); +} + +QString +TransformAligner::getAlignmentTransformName() +{ + QSettings settings; + settings.beginGroup("Alignment"); + TransformId id = + settings.value("transform-id", + "vamp:match-vamp-plugin:match:path").toString(); + settings.endGroup(); + return id; +} + +QString +TransformAligner::getTuningDifferenceTransformName() +{ + QSettings settings; + settings.beginGroup("Alignment"); + bool performPitchCompensation = + settings.value("align-pitch-aware", false).toBool(); + QString id = ""; + if (performPitchCompensation) { + id = settings.value + ("tuning-difference-transform-id", + "vamp:tuning-difference:tuning-difference:tuningfreq") + .toString(); + } + settings.endGroup(); + return id; +} + +bool +TransformAligner::isAvailable() +{ + TransformFactory *factory = TransformFactory::getInstance(); + TransformId id = getAlignmentTransformName(); + TransformId tdId = getTuningDifferenceTransformName(); + return factory->haveTransform(id) && + (tdId == "" || factory->haveTransform(tdId)); +} + +void +TransformAligner::begin() +{ + auto reference = + ModelById::getAs<RangeSummarisableTimeValueModel>(m_reference); + auto other = + ModelById::getAs<RangeSummarisableTimeValueModel>(m_toAlign); + + if (!reference || !other) return; + + // This involves creating a number of new models: + // + // 1. an AggregateWaveModel to provide the mixdowns of the main + // model and the new model in its two channels, as input to the + // MATCH plugin. We just call this one aggregateModel + // + // 2a. a SparseTimeValueModel which will be automatically created + // by FeatureExtractionModelTransformer when running the + // TuningDifference plugin to receive the relative tuning of the + // second model (if pitch-aware alignment is enabled in the + // preferences). This is m_tuningDiffOutputModel. + // + // 2b. a SparseTimeValueModel which will be automatically created + // by FeatureExtractionPluginTransformer when running the MATCH + // plugin to perform alignment (so containing the alignment path). + // This is m_pathOutputModel. + // + // 2c. a SparseTimeValueModel used solely to provide faked + // completion information to the AlignmentModel while a + // TuningDifference calculation is going on. We call this + // m_tuningDiffProgressModel. + // + // 3. an AlignmentModel, which stores the path and carries out + // alignment lookups on it. This one is m_alignmentModel. + // + // Models 1 and 3 are registered with the document, which will + // eventually release them. We don't release them here except in + // the case where an activity fails before the point where we + // would otherwise have registered them with the document. + // + // Models 2a (m_tuningDiffOutputModel), 2b (m_pathOutputModel) and + // 2c (m_tuningDiffProgressModel) are not registered with the + // document, because they are not intended to persist, and also + // Model 2c (m_tuningDiffProgressModel) is a bodge that we are + // embarrassed about, so we try to manage it ourselves without + // anyone else noticing. These have to be released by us when + // finished with, but their lifespans do not extend beyond the end + // of the alignment procedure, so this should be ok. + + AggregateWaveModel::ChannelSpecList components; + components.push_back + (AggregateWaveModel::ModelChannelSpec(m_reference, -1)); + + components.push_back + (AggregateWaveModel::ModelChannelSpec(m_toAlign, -1)); + + auto aggregateModel = std::make_shared<AggregateWaveModel>(components); + m_aggregateModel = ModelById::add(aggregateModel); + m_document->addNonDerivedModel(m_aggregateModel); + + auto alignmentModel = std::make_shared<AlignmentModel> + (m_reference, m_toAlign, ModelId()); + m_alignmentModel = ModelById::add(alignmentModel); + + TransformId tdId = getTuningDifferenceTransformName(); + + if (tdId == "") { + + if (beginAlignmentPhase()) { + other->setAlignment(m_alignmentModel); + m_document->addNonDerivedModel(m_alignmentModel); + } else { + QString error = alignmentModel->getError(); + ModelById::release(alignmentModel); + emit failed(m_toAlign, error); + return; + } + + } else { + + // Have a tuning-difference transform id, so run it + // asynchronously first + + TransformFactory *tf = TransformFactory::getInstance(); + + Transform transform = tf->getDefaultTransformFor + (tdId, aggregateModel->getSampleRate()); + + transform.setParameter("maxduration", 60); + transform.setParameter("maxrange", 6); + transform.setParameter("finetuning", false); + + SVDEBUG << "TransformAligner: Tuning difference transform step size " << transform.getStepSize() << ", block size " << transform.getBlockSize() << endl; + + ModelTransformerFactory *mtf = ModelTransformerFactory::getInstance(); + + QString message; + m_tuningDiffOutputModel = mtf->transform(transform, + m_aggregateModel, + message); + + auto tuningDiffOutputModel = + ModelById::getAs<SparseTimeValueModel>(m_tuningDiffOutputModel); + if (!tuningDiffOutputModel) { + SVCERR << "Align::alignModel: ERROR: Failed to create tuning-difference output model (no Tuning Difference plugin?)" << endl; + ModelById::release(alignmentModel); + emit failed(m_toAlign, message); + return; + } + + other->setAlignment(m_alignmentModel); + m_document->addNonDerivedModel(m_alignmentModel); + + connect(tuningDiffOutputModel.get(), + SIGNAL(completionChanged(ModelId)), + this, SLOT(tuningDifferenceCompletionChanged(ModelId))); + + // This model exists only so that the AlignmentModel can get a + // completion value from somewhere while the tuning difference + // calculation is going on + auto progressModel = std::make_shared<SparseTimeValueModel> + (aggregateModel->getSampleRate(), 1); + m_tuningDiffProgressModel = ModelById::add(progressModel); + progressModel->setCompletion(0); + alignmentModel->setPathFrom(m_tuningDiffProgressModel); + } +} + +void +TransformAligner::tuningDifferenceCompletionChanged(ModelId tuningDiffOutputModelId) +{ + if (m_tuningDiffOutputModel.isNone()) { + // we're done, this is probably a spurious queued event + return; + } + + if (tuningDiffOutputModelId != m_tuningDiffOutputModel) { + SVCERR << "WARNING: TransformAligner::tuningDifferenceCompletionChanged: Model " + << tuningDiffOutputModelId + << " is not ours! (ours is " + << m_tuningDiffOutputModel << ")" << endl; + return; + } + + auto tuningDiffOutputModel = + ModelById::getAs<SparseTimeValueModel>(m_tuningDiffOutputModel); + if (!tuningDiffOutputModel) { + SVCERR << "WARNING: TransformAligner::tuningDifferenceCompletionChanged: Model " + << tuningDiffOutputModelId + << " not known as SparseTimeValueModel" << endl; + return; + } + + auto alignmentModel = ModelById::getAs<AlignmentModel>(m_alignmentModel); + if (!alignmentModel) { + SVCERR << "WARNING: TransformAligner::tuningDifferenceCompletionChanged:" + << "alignment model has disappeared" << endl; + return; + } + + int completion = 0; + bool done = tuningDiffOutputModel->isReady(&completion); + + SVDEBUG << "TransformAligner::tuningDifferenceCompletionChanged: model " + << m_tuningDiffOutputModel << ", completion = " << completion + << ", done = " << done << endl; + + if (!done) { + // This will be the completion the alignment model reports, + // before the alignment actually begins. It goes up from 0 to + // 99 (not 100!) and then back to 0 again when we start + // calculating the actual path in the following phase + int clamped = (completion == 100 ? 99 : completion); + auto progressModel = + ModelById::getAs<SparseTimeValueModel>(m_tuningDiffProgressModel); + if (progressModel) { + progressModel->setCompletion(clamped); + } + return; + } + + m_tuningFrequency = 440.f; + + if (!tuningDiffOutputModel->isEmpty()) { + m_tuningFrequency = tuningDiffOutputModel->getAllEvents()[0].getValue(); + SVCERR << "TransformAligner::tuningDifferenceCompletionChanged: Reported tuning frequency = " << m_tuningFrequency << endl; + } else { + SVCERR << "TransformAligner::tuningDifferenceCompletionChanged: No tuning frequency reported" << endl; + } + + ModelById::release(tuningDiffOutputModel); + m_tuningDiffOutputModel = {}; + + alignmentModel->setPathFrom({}); // replace m_tuningDiffProgressModel + ModelById::release(m_tuningDiffProgressModel); + m_tuningDiffProgressModel = {}; + + beginAlignmentPhase(); +} + +bool +TransformAligner::beginAlignmentPhase() +{ + TransformId id = getAlignmentTransformName(); + + SVDEBUG << "TransformAligner::beginAlignmentPhase: transform is " + << id << endl; + + TransformFactory *tf = TransformFactory::getInstance(); + + auto aggregateModel = + ModelById::getAs<AggregateWaveModel>(m_aggregateModel); + auto alignmentModel = + ModelById::getAs<AlignmentModel>(m_alignmentModel); + + if (!aggregateModel || !alignmentModel) { + SVCERR << "TransformAligner::alignModel: ERROR: One or other of the aggregate & alignment models has disappeared" << endl; + return false; + } + + Transform transform = tf->getDefaultTransformFor + (id, aggregateModel->getSampleRate()); + + transform.setStepSize(transform.getBlockSize()/2); + transform.setParameter("serialise", 1); + transform.setParameter("smooth", 0); + transform.setParameter("zonewidth", 40); + transform.setParameter("noise", true); + transform.setParameter("minfreq", 500); + + int cents = 0; + + if (m_tuningFrequency != 0.f) { + transform.setParameter("freq2", m_tuningFrequency); + + double centsOffset = 0.f; + int pitch = Pitch::getPitchForFrequency(m_tuningFrequency, + ¢sOffset); + cents = int(round((pitch - 69) * 100 + centsOffset)); + SVCERR << "TransformAligner: frequency " << m_tuningFrequency + << " yields cents offset " << centsOffset + << " and pitch " << pitch << " -> cents " << cents << endl; + } + + alignmentModel->setRelativePitch(cents); + + SVDEBUG << "TransformAligner: Alignment transform step size " + << transform.getStepSize() << ", block size " + << transform.getBlockSize() << endl; + + ModelTransformerFactory *mtf = ModelTransformerFactory::getInstance(); + + QString message; + m_pathOutputModel = mtf->transform + (transform, m_aggregateModel, message); + + if (m_pathOutputModel.isNone()) { + transform.setStepSize(0); + m_pathOutputModel = mtf->transform + (transform, m_aggregateModel, message); + } + + auto pathOutputModel = + ModelById::getAs<SparseTimeValueModel>(m_pathOutputModel); + + //!!! callers will need to be updated to get error from + //!!! alignment model after initial call + + if (!pathOutputModel) { + SVCERR << "TransformAligner: ERROR: Failed to create alignment path (no MATCH plugin?)" << endl; + alignmentModel->setError(message); + return false; + } + + pathOutputModel->setCompletion(0); + alignmentModel->setPathFrom(m_pathOutputModel); + + connect(alignmentModel.get(), SIGNAL(completionChanged(ModelId)), + this, SLOT(alignmentCompletionChanged(ModelId))); + + return true; +} + +void +TransformAligner::alignmentCompletionChanged(ModelId alignmentModelId) +{ + if (alignmentModelId != m_alignmentModel) { + SVCERR << "WARNING: TransformAligner::alignmentCompletionChanged: Model " + << alignmentModelId + << " is not ours! (ours is " + << m_alignmentModel << ")" << endl; + return; + } + + auto alignmentModel = ModelById::getAs<AlignmentModel>(m_alignmentModel); + + if (alignmentModel && alignmentModel->isReady()) { + + m_incomplete = false; + + ModelById::release(m_pathOutputModel); + m_pathOutputModel = {}; + + disconnect(alignmentModel.get(), + SIGNAL(completionChanged(ModelId)), + this, SLOT(alignmentCompletionChanged(ModelId))); + emit complete(m_alignmentModel); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/align/TransformAligner.h Wed May 13 14:10:47 2020 +0100 @@ -0,0 +1,61 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef SV_TRANSFORM_ALIGNER_H +#define SV_TRANSFORM_ALIGNER_H + +#include "Aligner.h" + +class AlignmentModel; +class Document; + +class TransformAligner : public Aligner +{ + Q_OBJECT + +public: + TransformAligner(Document *doc, + ModelId reference, + ModelId toAlign); + + // Destroy the aligner, cleanly cancelling any ongoing alignment + ~TransformAligner(); + + void begin() override; + + static bool isAvailable(); + +private slots: + void alignmentCompletionChanged(ModelId); + void tuningDifferenceCompletionChanged(ModelId); + +private: + static QString getAlignmentTransformName(); + static QString getTuningDifferenceTransformName(); + + bool beginAlignmentPhase(); + + Document *m_document; + ModelId m_reference; + ModelId m_toAlign; + ModelId m_aggregateModel; // an AggregateWaveModel + ModelId m_alignmentModel; // an AlignmentModel + ModelId m_tuningDiffProgressModel; // SparseTimeValueModel, unreg'd with doc + ModelId m_tuningDiffOutputModel; // SparseTimeValueModel, unreg'd with doc + ModelId m_pathOutputModel; // SparseTimeValueModel, unreg'd with doc + float m_tuningFrequency; + bool m_incomplete; +}; + +#endif
--- a/files.pri Thu Apr 30 14:48:14 2020 +0100 +++ b/files.pri Wed May 13 14:10:47 2020 +0100 @@ -1,5 +1,9 @@ SVAPP_HEADERS += \ + align/Align.h \ + align/Aligner.h \ + align/ExternalProgramAligner.h \ + align/TransformAligner.h \ audio/AudioCallbackPlaySource.h \ audio/AudioCallbackRecordTarget.h \ audio/AudioGenerator.h \ @@ -8,7 +12,6 @@ audio/EffectWrapper.h \ audio/PlaySpeedRangeMapper.h \ audio/TimeStretchWrapper.h \ - framework/Align.h \ framework/Document.h \ framework/MainWindowBase.h \ framework/OSCScript.h \ @@ -17,6 +20,9 @@ framework/VersionTester.h SVAPP_SOURCES += \ + align/Align.cpp \ + align/ExternalProgramAligner.cpp \ + align/TransformAligner.cpp \ audio/AudioCallbackPlaySource.cpp \ audio/AudioCallbackRecordTarget.cpp \ audio/AudioGenerator.cpp \ @@ -25,7 +31,6 @@ audio/EffectWrapper.cpp \ audio/PlaySpeedRangeMapper.cpp \ audio/TimeStretchWrapper.cpp \ - framework/Align.cpp \ framework/Document.cpp \ framework/MainWindowBase.cpp \ framework/SVFileReader.cpp \
--- a/framework/Align.cpp Thu Apr 30 14:48:14 2020 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,673 +0,0 @@ -/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ - -/* - Sonic Visualiser - An audio file viewer and annotation editor. - Centre for Digital Music, Queen Mary, University of London. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. See the file - COPYING included with this distribution for more information. -*/ - -#include "Align.h" -#include "Document.h" - -#include "data/model/WaveFileModel.h" -#include "data/model/ReadOnlyWaveFileModel.h" -#include "data/model/AggregateWaveModel.h" -#include "data/model/RangeSummarisableTimeValueModel.h" -#include "data/model/SparseTimeValueModel.h" -#include "data/model/AlignmentModel.h" - -#include "data/fileio/CSVFileReader.h" - -#include "transform/TransformFactory.h" -#include "transform/ModelTransformerFactory.h" -#include "transform/FeatureExtractionModelTransformer.h" - -#include <QProcess> -#include <QSettings> -#include <QApplication> - -bool -Align::alignModel(Document *doc, ModelId ref, ModelId other, QString &error) -{ - QSettings settings; - settings.beginGroup("Preferences"); - bool useProgram = settings.value("use-external-alignment", false).toBool(); - QString program = settings.value("external-alignment-program", "").toString(); - settings.endGroup(); - - if (useProgram && (program != "")) { - return alignModelViaProgram(doc, ref, other, program, error); - } else { - return alignModelViaTransform(doc, ref, other, error); - } -} - -QString -Align::getAlignmentTransformName() -{ - QSettings settings; - settings.beginGroup("Alignment"); - TransformId id = - settings.value("transform-id", - "vamp:match-vamp-plugin:match:path").toString(); - settings.endGroup(); - return id; -} - -QString -Align::getTuningDifferenceTransformName() -{ - QSettings settings; - settings.beginGroup("Alignment"); - bool performPitchCompensation = - settings.value("align-pitch-aware", false).toBool(); - QString id = ""; - if (performPitchCompensation) { - id = settings.value - ("tuning-difference-transform-id", - "vamp:tuning-difference:tuning-difference:tuningfreq") - .toString(); - } - settings.endGroup(); - return id; -} - -bool -Align::canAlign() -{ - TransformFactory *factory = TransformFactory::getInstance(); - TransformId id = getAlignmentTransformName(); - TransformId tdId = getTuningDifferenceTransformName(); - return factory->haveTransform(id) && - (tdId == "" || factory->haveTransform(tdId)); -} - -void -Align::abandonOngoingAlignment(ModelId otherId) -{ - auto other = ModelById::getAs<RangeSummarisableTimeValueModel>(otherId); - if (!other) { - return; - } - - ModelId alignmentModelId = other->getAlignment(); - if (alignmentModelId.isNone()) { - return; - } - - SVCERR << "Align::abandonOngoingAlignment: An alignment is ongoing for model " - << otherId << " (alignment model id " << alignmentModelId - << "), abandoning it..." << endl; - - other->setAlignment({}); - - for (auto pp: m_pendingProcesses) { - if (alignmentModelId == pp.second) { - QProcess *process = pp.first; - m_pendingProcesses.erase(process); - SVCERR << "Align::abandonOngoingAlignment: Killing external " - << "alignment process " << process << "..." << endl; - delete process; // kills the process itself - break; - } - } - - if (m_pendingAlignments.find(alignmentModelId) != - m_pendingAlignments.end()) { - SVCERR << "Align::abandonOngoingAlignment: Releasing path output model " - << m_pendingAlignments[alignmentModelId] - << "..." << endl; - ModelById::release(m_pendingAlignments[alignmentModelId]); - SVCERR << "Align::abandonOngoingAlignment: Dropping alignment model " - << alignmentModelId - << " from pending alignments..." << endl; - m_pendingAlignments.erase(alignmentModelId); - } - - for (auto ptd: m_pendingTuningDiffs) { - if (alignmentModelId == ptd.second.alignment) { - SVCERR << "Align::abandonOngoingAlignment: Releasing preparatory model " - << ptd.second.preparatory << "..." << endl; - ModelById::release(ptd.second.preparatory); - SVCERR << "Align::abandonOngoingAlignment: Releasing pending tuning-diff model " - << ptd.first << "..." << endl; - ModelById::release(ptd.first); - SVCERR << "Align::abandonOngoingAlignment: Dropping tuning-diff model " - << ptd.first - << " from pending tuning diffs..." << endl; - m_pendingTuningDiffs.erase(ptd.first); - break; - } - } - - SVCERR << "Align::abandonOngoingAlignment: done" << endl; -} - -bool -Align::alignModelViaTransform(Document *doc, - ModelId referenceId, - ModelId otherId, - QString &error) -{ - QMutexLocker locker (&m_mutex); - - auto reference = - ModelById::getAs<RangeSummarisableTimeValueModel>(referenceId); - auto other = - ModelById::getAs<RangeSummarisableTimeValueModel>(otherId); - - if (!reference || !other) return false; - - // There may be an alignment already happening; we should stop it, - // which we can do by discarding the output models for its - // transforms - abandonOngoingAlignment(otherId); - - // This involves creating a number of new models: - // - // 1. an AggregateWaveModel to provide the mixdowns of the main - // model and the new model in its two channels, as input to the - // MATCH plugin. We just call this one aggregateModel - // - // 2a. a SparseTimeValueModel which will be automatically created - // by FeatureExtractionModelTransformer when running the - // TuningDifference plugin to receive the relative tuning of the - // second model (if pitch-aware alignment is enabled in the - // preferences). We call this tuningDiffOutputModel. - // - // 2b. a SparseTimeValueModel which will be automatically created - // by FeatureExtractionPluginTransformer when running the MATCH - // plugin to perform alignment (so containing the alignment path). - // We call this one pathOutputModel. - // - // 2c. a SparseTimeValueModel used solely to provide faked - // completion information to the AlignmentModel while a - // TuningDifference calculation is going on. We call this - // preparatoryModel. - // - // 3. an AlignmentModel, which stores the path and carries out - // alignment lookups on it. We just call this one alignmentModel. - // - // Models 1 and 3 are registered with the document, which will - // eventually release them. We don't release them here except in - // the case where an activity fails before the point where we - // would otherwise have registered them with the document. - // - // Models 2a (tuningDiffOutputModel), 2b (pathOutputModel) and 2c - // (preparatoryModel) are not registered with the document. Model - // 2b (pathOutputModel) is not registered because we do not have a - // stable reference to the document at the point where it is - // created. Model 2c (preparatoryModel) is not registered because - // it is a bodge that we are embarrassed about, so we try to - // manage it ourselves without anyone else noticing. Model 2a is - // not registered for symmetry with the other two. These have to - // be released by us when finished with, but their lifespans do - // not extend beyond the end of the alignment procedure, so this - // should be ok. - - AggregateWaveModel::ChannelSpecList components; - - components.push_back - (AggregateWaveModel::ModelChannelSpec(referenceId, -1)); - - components.push_back - (AggregateWaveModel::ModelChannelSpec(otherId, -1)); - - auto aggregateModel = std::make_shared<AggregateWaveModel>(components); - auto aggregateModelId = ModelById::add(aggregateModel); - doc->addNonDerivedModel(aggregateModelId); - - auto alignmentModel = std::make_shared<AlignmentModel> - (referenceId, otherId, ModelId()); - auto alignmentModelId = ModelById::add(alignmentModel); - - TransformId tdId = getTuningDifferenceTransformName(); - - if (tdId == "") { - - if (beginTransformDrivenAlignment(aggregateModelId, - alignmentModelId)) { - other->setAlignment(alignmentModelId); - doc->addNonDerivedModel(alignmentModelId); - } else { - error = alignmentModel->getError(); - ModelById::release(alignmentModel); - return false; - } - - } else { - - // Have a tuning-difference transform id, so run it - // asynchronously first - - TransformFactory *tf = TransformFactory::getInstance(); - - Transform transform = tf->getDefaultTransformFor - (tdId, aggregateModel->getSampleRate()); - - transform.setParameter("maxduration", 60); - transform.setParameter("maxrange", 6); - transform.setParameter("finetuning", false); - - SVDEBUG << "Align::alignModel: Tuning difference transform step size " << transform.getStepSize() << ", block size " << transform.getBlockSize() << endl; - - ModelTransformerFactory *mtf = ModelTransformerFactory::getInstance(); - - QString message; - ModelId tuningDiffOutputModelId = mtf->transform(transform, - aggregateModelId, - message); - - auto tuningDiffOutputModel = - ModelById::getAs<SparseTimeValueModel>(tuningDiffOutputModelId); - if (!tuningDiffOutputModel) { - SVCERR << "Align::alignModel: ERROR: Failed to create tuning-difference output model (no Tuning Difference plugin?)" << endl; - error = message; - ModelById::release(alignmentModel); - return false; - } - - other->setAlignment(alignmentModelId); - doc->addNonDerivedModel(alignmentModelId); - - connect(tuningDiffOutputModel.get(), - SIGNAL(completionChanged(ModelId)), - this, SLOT(tuningDifferenceCompletionChanged(ModelId))); - - TuningDiffRec rec; - rec.input = aggregateModelId; - rec.alignment = alignmentModelId; - - // This model exists only so that the AlignmentModel can get a - // completion value from somewhere while the tuning difference - // calculation is going on - auto preparatoryModel = std::make_shared<SparseTimeValueModel> - (aggregateModel->getSampleRate(), 1); - auto preparatoryModelId = ModelById::add(preparatoryModel); - preparatoryModel->setCompletion(0); - rec.preparatory = preparatoryModelId; - alignmentModel->setPathFrom(rec.preparatory); - - m_pendingTuningDiffs[tuningDiffOutputModelId] = rec; - - SVDEBUG << "Align::alignModelViaTransform: Made a note of pending tuning diff output model id " << tuningDiffOutputModelId << " with input " << rec.input << ", alignment model " << rec.alignment << ", preparatory model " << rec.preparatory << endl; - } - - return true; -} - -void -Align::tuningDifferenceCompletionChanged(ModelId tuningDiffOutputModelId) -{ - QMutexLocker locker(&m_mutex); - - if (m_pendingTuningDiffs.find(tuningDiffOutputModelId) == - m_pendingTuningDiffs.end()) { - SVDEBUG << "NOTE: Align::tuningDifferenceCompletionChanged: Model " - << tuningDiffOutputModelId - << " not found in pending tuning diff map, presuming " - << "completed or abandoned" << endl; - return; - } - - auto tuningDiffOutputModel = - ModelById::getAs<SparseTimeValueModel>(tuningDiffOutputModelId); - if (!tuningDiffOutputModel) { - SVCERR << "WARNING: Align::tuningDifferenceCompletionChanged: Model " - << tuningDiffOutputModelId - << " not known as SparseTimeValueModel" << endl; - return; - } - - TuningDiffRec rec = m_pendingTuningDiffs[tuningDiffOutputModelId]; - - auto alignmentModel = ModelById::getAs<AlignmentModel>(rec.alignment); - if (!alignmentModel) { - SVCERR << "WARNING: Align::tuningDifferenceCompletionChanged:" - << "alignment model has disappeared" << endl; - return; - } - - int completion = 0; - bool done = tuningDiffOutputModel->isReady(&completion); - - if (!done) { - // This will be the completion the alignment model reports, - // before the alignment actually begins. It goes up from 0 to - // 99 (not 100!) and then back to 0 again when we start - // calculating the actual path in the following phase - int clamped = (completion == 100 ? 99 : completion); - auto preparatoryModel = - ModelById::getAs<SparseTimeValueModel>(rec.preparatory); - if (preparatoryModel) { - preparatoryModel->setCompletion(clamped); - } - return; - } - - float tuningFrequency = 440.f; - - if (!tuningDiffOutputModel->isEmpty()) { - tuningFrequency = tuningDiffOutputModel->getAllEvents()[0].getValue(); - SVCERR << "Align::tuningDifferenceCompletionChanged: Reported tuning frequency = " << tuningFrequency << endl; - } else { - SVCERR << "Align::tuningDifferenceCompletionChanged: No tuning frequency reported" << endl; - } - - ModelById::release(tuningDiffOutputModel); - - alignmentModel->setPathFrom({}); // replace preparatoryModel - ModelById::release(rec.preparatory); - rec.preparatory = {}; - - m_pendingTuningDiffs.erase(tuningDiffOutputModelId); - - SVDEBUG << "Align::tuningDifferenceCompletionChanged: Erasing model " - << tuningDiffOutputModelId << " from pending tuning diffs and " - << "launching the alignment phase for alignment model " - << rec.alignment << " with tuning frequency " - << tuningFrequency << endl; - - beginTransformDrivenAlignment - (rec.input, rec.alignment, tuningFrequency); -} - -bool -Align::beginTransformDrivenAlignment(ModelId aggregateModelId, - ModelId alignmentModelId, - float tuningFrequency) -{ - TransformId id = getAlignmentTransformName(); - - TransformFactory *tf = TransformFactory::getInstance(); - - auto aggregateModel = - ModelById::getAs<AggregateWaveModel>(aggregateModelId); - auto alignmentModel = - ModelById::getAs<AlignmentModel>(alignmentModelId); - - if (!aggregateModel || !alignmentModel) { - SVCERR << "Align::alignModel: ERROR: One or other of the aggregate & alignment models has disappeared" << endl; - return false; - } - - Transform transform = tf->getDefaultTransformFor - (id, aggregateModel->getSampleRate()); - - transform.setStepSize(transform.getBlockSize()/2); - transform.setParameter("serialise", 1); - transform.setParameter("smooth", 0); - transform.setParameter("zonewidth", 40); - transform.setParameter("noise", true); - transform.setParameter("minfreq", 500); - - int cents = 0; - - if (tuningFrequency != 0.f) { - transform.setParameter("freq2", tuningFrequency); - - double centsOffset = 0.f; - int pitch = Pitch::getPitchForFrequency(tuningFrequency, ¢sOffset); - cents = int(round((pitch - 69) * 100 + centsOffset)); - SVCERR << "frequency " << tuningFrequency << " yields cents offset " << centsOffset << " and pitch " << pitch << " -> cents " << cents << endl; - } - - alignmentModel->setRelativePitch(cents); - - SVDEBUG << "Align::alignModel: Alignment transform step size " << transform.getStepSize() << ", block size " << transform.getBlockSize() << endl; - - ModelTransformerFactory *mtf = ModelTransformerFactory::getInstance(); - - QString message; - ModelId pathOutputModelId = mtf->transform - (transform, aggregateModelId, message); - - if (pathOutputModelId.isNone()) { - transform.setStepSize(0); - pathOutputModelId = mtf->transform - (transform, aggregateModelId, message); - } - - auto pathOutputModel = - ModelById::getAs<SparseTimeValueModel>(pathOutputModelId); - - //!!! callers will need to be updated to get error from - //!!! alignment model after initial call - - if (!pathOutputModel) { - SVCERR << "Align::alignModel: ERROR: Failed to create alignment path (no MATCH plugin?)" << endl; - alignmentModel->setError(message); - return false; - } - - pathOutputModel->setCompletion(0); - alignmentModel->setPathFrom(pathOutputModelId); - - m_pendingAlignments[alignmentModelId] = pathOutputModelId; - - connect(alignmentModel.get(), SIGNAL(completionChanged(ModelId)), - this, SLOT(alignmentCompletionChanged(ModelId))); - - return true; -} - -void -Align::alignmentCompletionChanged(ModelId alignmentModelId) -{ - QMutexLocker locker (&m_mutex); - - auto alignmentModel = ModelById::getAs<AlignmentModel>(alignmentModelId); - - if (alignmentModel && alignmentModel->isReady()) { - - if (m_pendingAlignments.find(alignmentModelId) != - m_pendingAlignments.end()) { - ModelId pathOutputModelId = m_pendingAlignments[alignmentModelId]; - ModelById::release(pathOutputModelId); - m_pendingAlignments.erase(alignmentModelId); - } - - disconnect(alignmentModel.get(), - SIGNAL(completionChanged(ModelId)), - this, SLOT(alignmentCompletionChanged(ModelId))); - emit alignmentComplete(alignmentModelId); - } -} - -bool -Align::alignModelViaProgram(Document *doc, - ModelId referenceId, - ModelId otherId, - QString program, - QString &error) -{ - // Run an external program, passing to it paths to the main - // model's audio file and the new model's audio file. It returns - // the path in CSV form through stdout. - - auto reference = ModelById::getAs<ReadOnlyWaveFileModel>(referenceId); - auto other = ModelById::getAs<ReadOnlyWaveFileModel>(otherId); - if (!reference || !other) { - SVCERR << "ERROR: Align::alignModelViaProgram: Can't align non-read-only models via program (no local filename available)" << endl; - return false; - } - - while (!reference->isReady(nullptr) || !other->isReady(nullptr)) { - qApp->processEvents(); - } - - QString refPath = reference->getLocalFilename(); - if (refPath == "") { - refPath = FileSource(reference->getLocation()).getLocalFilename(); - } - - QString otherPath = other->getLocalFilename(); - if (otherPath == "") { - otherPath = FileSource(other->getLocation()).getLocalFilename(); - } - - if (refPath == "" || otherPath == "") { - error = "Failed to find local filepath for wave-file model"; - return false; - } - - QProcess *process = nullptr; - ModelId alignmentModelId = {}; - - { - QMutexLocker locker (&m_mutex); - - auto alignmentModel = - std::make_shared<AlignmentModel>(referenceId, otherId, ModelId()); - - alignmentModelId = ModelById::add(alignmentModel); - other->setAlignment(alignmentModelId); - - process = new QProcess; - process->setProcessChannelMode(QProcess::ForwardedErrorChannel); - - connect(process, - SIGNAL(finished(int, QProcess::ExitStatus)), - this, - SLOT(alignmentProgramFinished(int, QProcess::ExitStatus))); - - m_pendingProcesses[process] = alignmentModelId; - } - - QStringList args; - args << refPath << otherPath; - - SVCERR << "Align::alignModelViaProgram: Starting program \"" - << program << "\" with args: "; - for (auto a: args) { - SVCERR << "\"" << a << "\" "; - } - SVCERR << endl; - - process->start(program, args); - - bool success = process->waitForStarted(); - - { - QMutexLocker locker(&m_mutex); - - if (!success) { - - SVCERR << "ERROR: Align::alignModelViaProgram: " - << "Program did not start" << endl; - error = "Alignment program \"" + program + "\" did not start"; - - m_pendingProcesses.erase(process); - other->setAlignment({}); - ModelById::release(alignmentModelId); - delete process; - - } else { - doc->addNonDerivedModel(alignmentModelId); - } - } - - return success; -} - -void -Align::alignmentProgramFinished(int exitCode, QProcess::ExitStatus status) -{ - QMutexLocker locker (&m_mutex); - - SVCERR << "Align::alignmentProgramFinished" << endl; - - QProcess *process = qobject_cast<QProcess *>(sender()); - - if (m_pendingProcesses.find(process) == m_pendingProcesses.end()) { - SVCERR << "ERROR: Align::alignmentProgramFinished: Process " << process - << " not found in process model map!" << endl; - return; - } - - ModelId alignmentModelId = m_pendingProcesses[process]; - auto alignmentModel = ModelById::getAs<AlignmentModel>(alignmentModelId); - if (!alignmentModel) return; - - if (exitCode == 0 && status == 0) { - - CSVFormat format; - format.setModelType(CSVFormat::TwoDimensionalModel); - format.setTimingType(CSVFormat::ExplicitTiming); - format.setTimeUnits(CSVFormat::TimeSeconds); - format.setColumnCount(2); - // The output format has time in the reference file first, and - // time in the "other" file in the second column. This is a - // more natural approach for a command-line alignment tool, - // but it's the opposite of what we expect for native - // alignment paths, which map from "other" file to - // reference. These column purpose settings reflect that. - format.setColumnPurpose(1, CSVFormat::ColumnStartTime); - format.setColumnPurpose(0, CSVFormat::ColumnValue); - format.setAllowQuoting(false); - format.setSeparator(','); - - CSVFileReader reader(process, format, alignmentModel->getSampleRate()); - if (!reader.isOK()) { - SVCERR << "ERROR: Align::alignmentProgramFinished: Failed to parse output" - << endl; - alignmentModel->setError - (QString("Failed to parse output of program: %1") - .arg(reader.getError())); - goto done; - } - - //!!! to use ById? - - Model *csvOutput = reader.load(); - - SparseTimeValueModel *path = - qobject_cast<SparseTimeValueModel *>(csvOutput); - if (!path) { - SVCERR << "ERROR: Align::alignmentProgramFinished: Output did not convert to sparse time-value model" - << endl; - alignmentModel->setError - ("Output of program did not produce sparse time-value model"); - delete csvOutput; - goto done; - } - - if (path->isEmpty()) { - SVCERR << "ERROR: Align::alignmentProgramFinished: Output contained no mappings" - << endl; - alignmentModel->setError - ("Output of alignment program contained no mappings"); - delete path; - goto done; - } - - SVCERR << "Align::alignmentProgramFinished: Setting alignment path (" - << path->getEventCount() << " point(s))" << endl; - - auto pathId = - ModelById::add(std::shared_ptr<SparseTimeValueModel>(path)); - alignmentModel->setPathFrom(pathId); - - emit alignmentComplete(alignmentModelId); - - ModelById::release(pathId); - - } else { - SVCERR << "ERROR: Align::alignmentProgramFinished: Aligner program " - << "failed: exit code " << exitCode << ", status " << status - << endl; - alignmentModel->setError - ("Aligner process returned non-zero exit status"); - } - -done: - m_pendingProcesses.erase(process); - delete process; -} -
--- a/framework/Align.h Thu Apr 30 14:48:14 2020 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,131 +0,0 @@ -/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ - -/* - Sonic Visualiser - An audio file viewer and annotation editor. - Centre for Digital Music, Queen Mary, University of London. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. See the file - COPYING included with this distribution for more information. -*/ - -#ifndef ALIGN_H -#define ALIGN_H - -#include <QString> -#include <QObject> -#include <QProcess> -#include <QMutex> -#include <set> - -#include "data/model/Model.h" - -class AlignmentModel; -class SparseTimeValueModel; -class AggregateWaveModel; -class Document; - -class Align : public QObject -{ - Q_OBJECT - -public: - Align() { } - - /** - * Align the "other" model to the reference, attaching an - * AlignmentModel to it. Alignment is carried out by the method - * configured in the user preferences (either a plugin transform - * or an external process) and is done asynchronously. - * - * The return value indicates whether the alignment procedure - * started successfully. If it is true, then an AlignmentModel has - * been constructed and attached to the toAlign model, and you can - * query that model to discover the alignment progress, eventual - * outcome, and any error message generated during alignment. (The - * AlignmentModel is subsequently owned by the toAlign model.) - * Conversely if alignModel returns false, no AlignmentModel has - * been created, and the error return argument will contain an - * error report about whatever problem prevented this from - * happening. - * - * A single Align object may carry out many simultanous alignment - * calls -- you do not need to create a new Align object each - * time, nor to wait for an alignment to be complete before - * starting a new one. - * - * The Align object must survive after this call, for at least as - * long as the alignment takes. The usual expectation is that the - * Align object will simply share the process or document - * lifespan. - */ - bool alignModel(Document *doc, - ModelId reference, - ModelId toAlign, - QString &error); - - bool alignModelViaTransform(Document *doc, - ModelId reference, - ModelId toAlign, - QString &error); - - bool alignModelViaProgram(Document *doc, - ModelId reference, - ModelId toAlign, - QString program, - QString &error); - - /** - * Return true if the alignment facility is available (relevant - * plugin installed, etc). - */ - static bool canAlign(); - -signals: - /** - * Emitted when an alignment is successfully completed. The - * reference and other models can be queried from the alignment - * model. - */ - void alignmentComplete(ModelId alignmentModel); // an AlignmentModel - -private slots: - void alignmentCompletionChanged(ModelId); - void tuningDifferenceCompletionChanged(ModelId); - void alignmentProgramFinished(int, QProcess::ExitStatus); - -private: - static QString getAlignmentTransformName(); - static QString getTuningDifferenceTransformName(); - - bool beginTransformDrivenAlignment(ModelId, // an AggregateWaveModel - ModelId, // an AlignmentModel - float tuningFrequency = 0.f); - - void abandonOngoingAlignment(ModelId otherId); - - QMutex m_mutex; - - struct TuningDiffRec { - ModelId input; // an AggregateWaveModel - ModelId alignment; // an AlignmentModel - ModelId preparatory; // a SparseTimeValueModel - }; - - // tuning-difference output model (a SparseTimeValueModel) -> data - // needed for subsequent alignment - std::map<ModelId, TuningDiffRec> m_pendingTuningDiffs; - - // alignment model id -> path output model id - std::map<ModelId, ModelId> m_pendingAlignments; - - // external alignment subprocess -> model into which to stuff the - // results (an AlignmentModel) - std::map<QProcess *, ModelId> m_pendingProcesses; -}; - -#endif -
--- a/framework/Document.cpp Thu Apr 30 14:48:14 2020 +0100 +++ b/framework/Document.cpp Wed May 13 14:10:47 2020 +0100 @@ -15,8 +15,6 @@ #include "Document.h" -#include "Align.h" - #include "data/model/WaveFileModel.h" #include "data/model/WritableWaveFileModel.h" #include "data/model/DenseThreeDimensionalModel.h" @@ -39,7 +37,7 @@ #include <typeinfo> #include "data/model/AlignmentModel.h" -#include "Align.h" +#include "align/Align.h" using std::vector; @@ -59,6 +57,9 @@ connect(m_align, SIGNAL(alignmentComplete(ModelId)), this, SIGNAL(alignmentComplete(ModelId))); + + connect(m_align, SIGNAL(alignmentFailed(ModelId, QString)), + this, SIGNAL(alignmentFailed(ModelId, QString))); } Document::~Document() @@ -557,7 +558,7 @@ } if (m_autoAlignment) { - SVDEBUG << "Document::setMainModel: auto-alignment is on, aligning model if possible" << endl; + SVDEBUG << "Document::setMainModel: auto-alignment is on, aligning main model if applicable" << endl; alignModel(m_mainModel); } else { SVDEBUG << "Document::setMainModel: auto-alignment is off" << endl; @@ -1147,11 +1148,7 @@ << endl; } - QString err; - if (!m_align->alignModel(this, m_mainModel, modelId, err)) { - SVCERR << "Alignment failed: " << err << endl; - emit alignmentFailed(err); - } + m_align->scheduleAlignment(this, m_mainModel, modelId); } void
--- a/framework/Document.h Thu Apr 30 14:48:14 2020 +0100 +++ b/framework/Document.h Wed May 13 14:10:47 2020 +0100 @@ -331,7 +331,7 @@ QString message); void alignmentComplete(ModelId); // an AlignmentModel - void alignmentFailed(QString message); + void alignmentFailed(ModelId, QString message); // an AlignmentModel void activity(QString);
--- a/framework/MainWindowBase.cpp Thu Apr 30 14:48:14 2020 +0100 +++ b/framework/MainWindowBase.cpp Wed May 13 14:10:47 2020 +0100 @@ -2726,8 +2726,8 @@ this, SLOT(modelRegenerationWarning(QString, QString, QString))); connect(m_document, SIGNAL(alignmentComplete(ModelId)), this, SLOT(alignmentComplete(ModelId))); - connect(m_document, SIGNAL(alignmentFailed(QString)), - this, SLOT(alignmentFailed(QString))); + connect(m_document, SIGNAL(alignmentFailed(ModelId, QString)), + this, SLOT(alignmentFailed(ModelId, QString))); m_document->setAutoAlignment(m_viewManager->getAlignMode());
--- a/framework/MainWindowBase.h Thu Apr 30 14:48:14 2020 +0100 +++ b/framework/MainWindowBase.h Wed May 13 14:10:47 2020 +0100 @@ -361,7 +361,7 @@ virtual void modelRegenerationWarning(QString, QString, QString) = 0; virtual void alignmentComplete(ModelId); - virtual void alignmentFailed(QString) = 0; + virtual void alignmentFailed(ModelId, QString) = 0; virtual void paneRightButtonMenuRequested(Pane *, QPoint point) = 0; virtual void panePropertiesRightButtonMenuRequested(Pane *, QPoint point) = 0;