Mercurial > hg > svcore
changeset 1522:b7cb203ee344
Merge from branch import-audio-data
| author | Chris Cannam |
|---|---|
| date | Wed, 12 Sep 2018 15:57:49 +0100 |
| parents | a250a54c11cc (current diff) 2d291eac9f21 (diff) |
| children | c1b2eab6ac51 |
| files | |
| diffstat | 20 files changed, 839 insertions(+), 99 deletions(-) [+] |
line wrap: on
line diff
--- a/base/ProgressReporter.h Tue Sep 04 11:31:35 2018 +0100 +++ b/base/ProgressReporter.h Wed Sep 12 15:57:49 2018 +0100 @@ -13,8 +13,8 @@ COPYING included with this distribution for more information. */ -#ifndef _PROGRESS_REPORTER_H_ -#define _PROGRESS_REPORTER_H_ +#ifndef SV_PROGRESS_REPORTER_H +#define SV_PROGRESS_REPORTER_H #include <QObject> #include <QString>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/base/RecordDirectory.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -0,0 +1,67 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "RecordDirectory.h" +#include "TempDirectory.h" + +#include <QDir> +#include <QDateTime> + +#include "Debug.h" + +QString +RecordDirectory::getRecordContainerDirectory() +{ + QDir parent(TempDirectory::getInstance()->getContainingPath()); + QString subdirname("recorded"); + + if (!parent.mkpath(subdirname)) { + SVCERR << "ERROR: RecordDirectory::getRecordContainerDirectory: Failed to create recorded dir in \"" << parent.canonicalPath() << "\"" << endl; + return ""; + } else { + return parent.filePath(subdirname); + } +} + +QString +RecordDirectory::getRecordDirectory() +{ + QDir parent(getRecordContainerDirectory()); + QDateTime now = QDateTime::currentDateTime(); + QString subdirname = QString("%1").arg(now.toString("yyyyMMdd")); + + if (!parent.mkpath(subdirname)) { + SVCERR << "ERROR: RecordDirectory::getRecordDirectory: Failed to create recorded dir in \"" << parent.canonicalPath() << "\"" << endl; + return ""; + } else { + return parent.filePath(subdirname); + } +} + +QString +RecordDirectory::getConvertedAudioDirectory() +{ + QDir parent(getRecordContainerDirectory()); + QDateTime now = QDateTime::currentDateTime(); + QString subdirname = "converted"; + + if (!parent.mkpath(subdirname)) { + SVCERR << "ERROR: RecordDirectory::getConvertedAudioDirectory: Failed to create recorded dir in \"" << parent.canonicalPath() << "\"" << endl; + return ""; + } else { + return parent.filePath(subdirname); + } +} + +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/base/RecordDirectory.h Wed Sep 12 15:57:49 2018 +0100 @@ -0,0 +1,59 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Sonic Visualiser + An audio file viewer and annotation editor. + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef SV_RECORD_DIRECTORY_H +#define SV_RECORD_DIRECTORY_H + +#include <QString> + +/** + * Report the intended target location for recorded audio files. + */ +class RecordDirectory +{ +public: + /** + * Return the directory in which a recorded file should be saved. + * This may vary depending on the current date and time, and so + * should be queried afresh for each recording. The directory will + * also be created if it does not yet exist. + * + * Returns an empty string if the record directory did not exist + * and could not be created. + */ + static QString getRecordDirectory(); + + /** + * Return the root "recorded files" directory. If + * getRecordDirectory() is returning a datestamped directory, then + * this will be its parent. The directory will also be created if + * it does not yet exist. + * + * Returns an empty string if the record directory did not exist + * and could not be created. + */ + static QString getRecordContainerDirectory(); + + /** + * Return the directory in which an audio file converted from a + * data file should be saved. The directory will also be created if + * it does not yet exist. + * + * Returns an empty string if the directory did not exist and + * could not be created. + */ + static QString getConvertedAudioDirectory(); +}; + +#endif
--- a/base/TempDirectory.h Tue Sep 04 11:31:35 2018 +0100 +++ b/base/TempDirectory.h Wed Sep 12 15:57:49 2018 +0100 @@ -13,8 +13,8 @@ COPYING included with this distribution for more information. */ -#ifndef _TEMP_DIRECTORY_H_ -#define _TEMP_DIRECTORY_H_ +#ifndef SV_TEMP_DIRECTORY_H +#define SV_TEMP_DIRECTORY_H #include <QString> #include <QMutex>
--- a/data/fileio/CSVFileReader.cpp Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/CSVFileReader.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -18,19 +18,24 @@ #include "model/Model.h" #include "base/RealTime.h" #include "base/StringBits.h" +#include "base/ProgressReporter.h" +#include "base/RecordDirectory.h" #include "model/SparseOneDimensionalModel.h" #include "model/SparseTimeValueModel.h" #include "model/EditableDenseThreeDimensionalModel.h" #include "model/RegionModel.h" #include "model/NoteModel.h" +#include "model/WritableWaveFileModel.h" #include "DataFileReaderFactory.h" #include <QFile> +#include <QDir> #include <QFileInfo> #include <QString> #include <QRegExp> #include <QStringList> #include <QTextStream> +#include <QDateTime> #include <iostream> #include <map> @@ -39,12 +44,17 @@ using namespace std; CSVFileReader::CSVFileReader(QString path, CSVFormat format, - sv_samplerate_t mainModelSampleRate) : + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter) : m_format(format), m_device(0), m_ownDevice(true), m_warnings(0), - m_mainModelSampleRate(mainModelSampleRate) + m_mainModelSampleRate(mainModelSampleRate), + m_fileSize(0), + m_readCount(0), + m_progress(-1), + m_reporter(reporter) { QFile *file = new QFile(path); bool good = false; @@ -60,19 +70,27 @@ if (good) { m_device = file; m_filename = QFileInfo(path).fileName(); + m_fileSize = file->size(); + if (m_reporter) m_reporter->setDefinite(true); } else { delete file; } } CSVFileReader::CSVFileReader(QIODevice *device, CSVFormat format, - sv_samplerate_t mainModelSampleRate) : + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter) : m_format(format), m_device(device), m_ownDevice(false), m_warnings(0), - m_mainModelSampleRate(mainModelSampleRate) + m_mainModelSampleRate(mainModelSampleRate), + m_fileSize(0), + m_readCount(0), + m_progress(-1), + m_reporter(reporter) { + if (m_reporter) m_reporter->setDefinite(false); } CSVFileReader::~CSVFileReader() @@ -184,6 +202,7 @@ RegionModel *model2a = 0; NoteModel *model2b = 0; EditableDenseThreeDimensionalModel *model3 = 0; + WritableWaveFileModel *modelW = 0; Model *model = 0; QTextStream in(m_device); @@ -203,8 +222,6 @@ sv_frame_t startFrame = 0; // for calculation of dense model resolution bool firstEverValue = true; - - map<QString, int> labelCountMap; int valueColumns = 0; for (int i = 0; i < m_format.getColumnCount(); ++i) { @@ -213,7 +230,41 @@ } } - while (!in.atEnd()) { + int audioChannels = 0; + float **audioSamples = 0; + float sampleShift = 0.f; + float sampleScale = 1.f; + + if (modelType == CSVFormat::WaveFileModel) { + + audioChannels = valueColumns; + + audioSamples = + breakfastquay::allocate_and_zero_channels<float> + (audioChannels, 1); + + switch (m_format.getAudioSampleRange()) { + case CSVFormat::SampleRangeSigned1: + case CSVFormat::SampleRangeOther: + sampleShift = 0.f; + sampleScale = 1.f; + break; + case CSVFormat::SampleRangeUnsigned255: + sampleShift = -128.f; + sampleScale = 1.f / 128.f; + break; + case CSVFormat::SampleRangeSigned32767: + sampleShift = 0.f; + sampleScale = 1.f / 32768.f; + break; + } + } + + map<QString, int> labelCountMap; + + bool abandoned = false; + + while (!in.atEnd() && !abandoned) { // QTextStream's readLine doesn't cope with old-style Mac // CR-only line endings. Why did they bother making the class @@ -228,6 +279,26 @@ QString chunk = in.readLine(); QStringList lines = chunk.split('\r', QString::SkipEmptyParts); + + m_readCount += chunk.size() + 1; + + if (m_reporter) { + if (m_reporter->wasCancelled()) { + abandoned = true; + break; + } + int progress; + if (m_fileSize > 0) { + progress = int((double(m_readCount) / double(m_fileSize)) + * 100.0); + } else { + progress = int(m_readCount / 10000); + } + if (progress != m_progress) { + m_reporter->setProgress(progress); + m_progress = progress; + } + } for (int li = 0; li < lines.size(); ++li) { @@ -238,6 +309,8 @@ QStringList list = StringBits::split(line, separator, allowQuoting); if (!model) { + QString modelName = m_filename; + switch (modelType) { case CSVFormat::OneDimensionalModel: @@ -268,22 +341,50 @@ EditableDenseThreeDimensionalModel::NoCompression); model = model3; break; + + case CSVFormat::WaveFileModel: + { + bool normalise = (m_format.getAudioSampleRange() + == CSVFormat::SampleRangeOther); + QString path = getConvertedAudioFilePath(); + modelW = new WritableWaveFileModel + (path, sampleRate, valueColumns, + normalise ? + WritableWaveFileModel::Normalisation::Peak : + WritableWaveFileModel::Normalisation::None); + modelName = QFileInfo(path).fileName(); + model = modelW; + break; + } } - if (model) { - if (m_filename != "") { - model->setObjectName(m_filename); + if (model && model->isOK()) { + if (modelName != "") { + model->setObjectName(modelName); } } } + if (!model || !model->isOK()) { + SVCERR << "Failed to create model to load CSV file into" + << endl; + if (model) { + delete model; + model = 0; + model1 = 0; model2 = 0; model2a = 0; model2b = 0; + model3 = 0; modelW = 0; + } + abandoned = true; + break; + } + float value = 0.f; float pitch = 0.f; QString label = ""; duration = 0.f; haveEndTime = false; - + for (int i = 0; i < list.size(); ++i) { QString s = list[i]; @@ -402,8 +503,52 @@ // << frameNo << ", time " << RealTime::frame2RealTime(frameNo, sampleRate) << endl; model3->setColumn(lineno, values); + + } else if (modelType == CSVFormat::WaveFileModel) { + + int channel = 0; + + for (int i = 0; + i < list.size() && channel < audioChannels; + ++i) { + + if (m_format.getColumnPurpose(i) != + CSVFormat::ColumnValue) { + continue; + } + + bool ok = false; + float value = list[i].toFloat(&ok); + if (!ok) { + value = 0.f; + } + + value += sampleShift; + value *= sampleScale; + + audioSamples[channel][0] = value; + + ++channel; + } + + while (channel < audioChannels) { + audioSamples[channel][0] = 0.f; + ++channel; + } + + bool ok = modelW->addSamples(audioSamples, 1); + + if (!ok) { + if (warnings < warnLimit) { + SVCERR << "WARNING: CSVFileReader::load: " + << "Unable to add sample to wave-file model" + << endl; + SVCERR << line << endl; + ++warnings; + } + } } - + ++lineno; if (timingType == CSVFormat::ImplicitTiming || list.size() == 0) { @@ -479,6 +624,32 @@ model3->setMaximumLevel(max); } + if (modelW) { + breakfastquay::deallocate_channels(audioSamples, audioChannels); + modelW->updateModel(); + modelW->writeComplete(); + } + return model; } +QString +CSVFileReader::getConvertedAudioFilePath() const +{ + QString base = m_filename; + base.replace(QRegExp("[/\\,.:;~<>\"'|?%*]+"), "_"); + + QString convertedFileDir = RecordDirectory::getConvertedAudioDirectory(); + if (convertedFileDir == "") { + SVCERR << "WARNING: CSVFileReader::getConvertedAudioFilePath: Failed to retrieve converted audio directory" << endl; + return ""; + } + + auto ms = QDateTime::currentDateTime().toMSecsSinceEpoch(); + auto s = ms / 1000; // there is a toSecsSinceEpoch in Qt 5.8 but + // we currently want to support older versions + + return QDir(convertedFileDir).filePath + (QString("%1-%2.wav").arg(base).arg(s)); +} +
--- a/data/fileio/CSVFileReader.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/CSVFileReader.h Wed Sep 12 15:57:49 2018 +0100 @@ -27,6 +27,7 @@ #include <QIODevice> class QFile; +class ProgressReporter; class CSVFileReader : public DataFileReader { @@ -35,7 +36,9 @@ * Construct a CSVFileReader to read the CSV file at the given * path, with the given format. */ - CSVFileReader(QString path, CSVFormat format, sv_samplerate_t mainModelSampleRate); + CSVFileReader(QString path, CSVFormat format, + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); /** * Construct a CSVFileReader to read from the given @@ -43,7 +46,9 @@ * CSVFileReader will not close or delete it and it must outlive * the CSVFileReader. */ - CSVFileReader(QIODevice *device, CSVFormat format, sv_samplerate_t mainModelSampleRate); + CSVFileReader(QIODevice *device, CSVFormat format, + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); virtual ~CSVFileReader(); @@ -60,9 +65,15 @@ QString m_error; mutable int m_warnings; sv_samplerate_t m_mainModelSampleRate; + qint64 m_fileSize; + mutable qint64 m_readCount; + mutable int m_progress; + ProgressReporter *m_reporter; sv_frame_t convertTimeValue(QString, int lineno, sv_samplerate_t sampleRate, int windowSize) const; + + QString getConvertedAudioFilePath() const; };
--- a/data/fileio/CSVFormat.cpp Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/CSVFormat.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -71,17 +71,20 @@ for (int li = 0; li < lines.size(); ++li) { QString line = lines[li]; - if (line.startsWith("#") || line == "") continue; + if (line.startsWith("#") || line == "") { + continue; + } guessQualities(line, lineno); ++lineno; } - if (lineno >= 50) break; + if (lineno >= 150) break; } guessPurposes(); + guessAudioSampleRange(); } void @@ -91,6 +94,8 @@ for (int i = 0; i < int(sizeof(candidates)/sizeof(candidates[0])); ++i) { if (StringBits::split(line, candidates[i], m_allowQuoting).size() >= 2) { m_separator = candidates[i]; + SVDEBUG << "Estimated column separator: '" << m_separator + << "'" << endl; return; } } @@ -111,7 +116,8 @@ // something that indicates otherwise: ColumnQualities defaultQualities = - ColumnNumeric | ColumnIntegral | ColumnIncreasing | ColumnNearEmpty; + ColumnNumeric | ColumnIntegral | ColumnSmall | + ColumnIncreasing | ColumnNearEmpty; for (int i = 0; i < cols; ++i) { @@ -128,7 +134,9 @@ bool numeric = (qualities & ColumnNumeric); bool integral = (qualities & ColumnIntegral); bool increasing = (qualities & ColumnIncreasing); + bool small = (qualities & ColumnSmall); bool large = (qualities & ColumnLarge); // this one defaults to off + bool signd = (qualities & ColumnSigned); // also defaults to off bool emptyish = (qualities & ColumnNearEmpty); if (lineno > 1 && s.trimmed() != "") { @@ -145,7 +153,15 @@ value = (float)StringBits::stringToDoubleLocaleFree(s, &ok); } if (ok) { - if (lineno < 2 && value > 1000.f) large = true; + if (lineno < 2 && value > 1000.f) { + large = true; + } + if (value < 0.f) { + signd = true; + } + if (value < -1.f || value > 1.f) { + small = false; + } } else { numeric = false; } @@ -172,7 +188,9 @@ (numeric ? ColumnNumeric : 0) | (integral ? ColumnIntegral : 0) | (increasing ? ColumnIncreasing : 0) | + (small ? ColumnSmall : 0) | (large ? ColumnLarge : 0) | + (signd ? ColumnSigned : 0) | (emptyish ? ColumnNearEmpty : 0); } @@ -200,6 +218,12 @@ int timingColumnCount = 0; + SVDEBUG << "Estimated column qualities overall: "; + for (int i = 0; i < m_columnCount; ++i) { + SVDEBUG << int(m_columnQualities[i]) << " "; + } + SVDEBUG << endl; + // if our first column has zero or one entries in it and the rest // have more, then we'll default to ignoring the first column and // counting the next one as primary. (e.g. Sonic Annotator output @@ -328,6 +352,74 @@ SVDEBUG << "Estimated units: " << m_timeUnits << endl; } +void +CSVFormat::guessAudioSampleRange() +{ + AudioSampleRange range = SampleRangeSigned1; + + range = SampleRangeSigned1; + bool knownSigned = false; + bool knownNonIntegral = false; + + SVDEBUG << "CSVFormat::guessAudioSampleRange: starting with assumption of " + << range << endl; + + for (int i = 0; i < m_columnCount; ++i) { + if (m_columnPurposes[i] != ColumnValue) { + SVDEBUG << "... column " << i + << " is not apparently a value, ignoring" << endl; + continue; + } + if (!(m_columnQualities[i] & ColumnIntegral)) { + knownNonIntegral = true; + if (range == SampleRangeUnsigned255 || + range == SampleRangeSigned32767) { + range = SampleRangeOther; + } + SVDEBUG << "... column " << i + << " is non-integral, updating range to " << range << endl; + } + if (m_columnQualities[i] & ColumnLarge) { + if (range == SampleRangeSigned1 || + range == SampleRangeUnsigned255) { + if (knownNonIntegral) { + range = SampleRangeOther; + } else { + range = SampleRangeSigned32767; + } + } + SVDEBUG << "... column " << i << " is large, updating range to " + << range << endl; + } + if (m_columnQualities[i] & ColumnSigned) { + knownSigned = true; + if (range == SampleRangeUnsigned255) { + range = SampleRangeSigned32767; + } + SVDEBUG << "... column " << i << " is signed, updating range to " + << range << endl; + } + if (!(m_columnQualities[i] & ColumnSmall)) { + if (range == SampleRangeSigned1) { + if (knownNonIntegral) { + range = SampleRangeOther; + } else if (knownSigned) { + range = SampleRangeSigned32767; + } else { + range = SampleRangeUnsigned255; + } + } + SVDEBUG << "... column " << i << " is not small, updating range to " + << range << endl; + } + } + + SVDEBUG << "CSVFormat::guessAudioSampleRange: ended up with range " + << range << endl; + + m_audioSampleRange = range; +} + CSVFormat::ColumnPurpose CSVFormat::getColumnPurpose(int i) {
--- a/data/fileio/CSVFormat.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/CSVFormat.h Wed Sep 12 15:57:49 2018 +0100 @@ -29,7 +29,8 @@ TwoDimensionalModel, TwoDimensionalModelWithDuration, TwoDimensionalModelWithDurationAndPitch, - ThreeDimensionalModel + ThreeDimensionalModel, + WaveFileModel }; enum TimingType { @@ -55,14 +56,23 @@ }; enum ColumnQuality { - ColumnNumeric = 1, - ColumnIntegral = 2, - ColumnIncreasing = 4, - ColumnLarge = 8, - ColumnNearEmpty = 16, + ColumnNumeric = 1, // No non-numeric values were seen in sample + ColumnIntegral = 2, // All sampled values were integers + ColumnIncreasing = 4, // Sampled values were monotonically increasing + ColumnSmall = 8, // All sampled values had magnitude < 1 + ColumnLarge = 16, // Values "quickly" grew to over 1000 + ColumnSigned = 32, // Some negative values were seen + ColumnNearEmpty = 64, // Nothing in this column beyond first row }; typedef unsigned int ColumnQualities; + enum AudioSampleRange { + SampleRangeSigned1 = 0, // -1 .. 1 + SampleRangeUnsigned255, // 0 .. 255 + SampleRangeSigned32767, // -32768 .. 32767 + SampleRangeOther // Other/unknown: Normalise on load + }; + CSVFormat() : // arbitrary defaults m_modelType(TwoDimensionalModel), m_timingType(ExplicitTiming), @@ -72,6 +82,7 @@ m_windowSize(1024), m_columnCount(0), m_variableColumnCount(false), + m_audioSampleRange(SampleRangeOther), m_allowQuoting(true), m_maxExampleCols(0) { } @@ -93,6 +104,7 @@ sv_samplerate_t getSampleRate() const { return m_sampleRate; } int getWindowSize() const { return m_windowSize; } int getColumnCount() const { return m_columnCount; } + AudioSampleRange getAudioSampleRange() const { return m_audioSampleRange; } bool getAllowQuoting() const { return m_allowQuoting; } QChar getSeparator() const { if (m_separator == "") return ' '; @@ -106,6 +118,7 @@ void setSampleRate(sv_samplerate_t r) { m_sampleRate = r; } void setWindowSize(int s) { m_windowSize = s; } void setColumnCount(int c) { m_columnCount = c; } + void setAudioSampleRange(AudioSampleRange r) { m_audioSampleRange = r; } void setAllowQuoting(bool q) { m_allowQuoting = q; } QList<ColumnPurpose> getColumnPurposes() const { return m_columnPurposes; } @@ -116,10 +129,15 @@ void setColumnPurpose(int i, ColumnPurpose p); // read-only; only valid if format has been guessed: - QList<ColumnQualities> getColumnQualities() const { return m_columnQualities; } + const QList<ColumnQualities> &getColumnQualities() const { + return m_columnQualities; + } // read-only; only valid if format has been guessed: - QList<QStringList> getExample() const { return m_example; } + const QList<QStringList> &getExample() const { + return m_example; + } + int getMaxExampleCols() const { return m_maxExampleCols; } protected: @@ -136,6 +154,8 @@ QList<ColumnQualities> m_columnQualities; QList<ColumnPurpose> m_columnPurposes; + AudioSampleRange m_audioSampleRange; + QList<float> m_prevValues; bool m_allowQuoting; @@ -146,9 +166,7 @@ void guessSeparator(QString line); void guessQualities(QString line, int lineno); void guessPurposes(); - - void guessFormatFor_Old(QString path); - + void guessAudioSampleRange(); }; #endif
--- a/data/fileio/DataFileReaderFactory.cpp Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/DataFileReaderFactory.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -32,21 +32,28 @@ bool csv, MIDIFileImportPreferenceAcquirer *acquirer, CSVFormat format, - sv_samplerate_t mainModelSampleRate) + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter) { QString err; DataFileReader *reader = 0; if (!csv) { - reader = new MIDIFileReader(path, acquirer, mainModelSampleRate); + reader = new MIDIFileReader(path, + acquirer, + mainModelSampleRate, + reporter); if (reader->isOK()) return reader; if (reader->getError() != "") err = reader->getError(); delete reader; } if (csv) { - reader = new CSVFileReader(path, format, mainModelSampleRate); + reader = new CSVFileReader(path, + format, + mainModelSampleRate, + reporter); if (reader->isOK()) return reader; if (reader->getError() != "") err = reader->getError(); delete reader; @@ -58,14 +65,15 @@ DataFileReader * DataFileReaderFactory::createReader(QString path, MIDIFileImportPreferenceAcquirer *acquirer, - sv_samplerate_t mainModelSampleRate) + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter) { DataFileReader *reader = createReader - (path, false, acquirer, CSVFormat(), mainModelSampleRate); + (path, false, acquirer, CSVFormat(), mainModelSampleRate, reporter); if (reader) return reader; reader = createReader - (path, true, acquirer, CSVFormat(path), mainModelSampleRate); + (path, true, acquirer, CSVFormat(path), mainModelSampleRate, reporter); if (reader) return reader; return 0; @@ -74,11 +82,13 @@ Model * DataFileReaderFactory::load(QString path, MIDIFileImportPreferenceAcquirer *acquirer, - sv_samplerate_t mainModelSampleRate) + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter) { DataFileReader *reader = createReader(path, acquirer, - mainModelSampleRate); + mainModelSampleRate, + reporter); if (!reader) return NULL; try { @@ -94,12 +104,14 @@ Model * DataFileReaderFactory::loadNonCSV(QString path, MIDIFileImportPreferenceAcquirer *acquirer, - sv_samplerate_t mainModelSampleRate) + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter) { DataFileReader *reader = createReader(path, false, acquirer, CSVFormat(), - mainModelSampleRate); + mainModelSampleRate, + reporter); if (!reader) return NULL; try { @@ -114,10 +126,12 @@ Model * DataFileReaderFactory::loadCSV(QString path, CSVFormat format, - sv_samplerate_t mainModelSampleRate) + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter) { DataFileReader *reader = createReader(path, true, 0, format, - mainModelSampleRate); + mainModelSampleRate, + reporter); if (!reader) return NULL; try {
--- a/data/fileio/DataFileReaderFactory.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/DataFileReaderFactory.h Wed Sep 12 15:57:49 2018 +0100 @@ -13,8 +13,8 @@ COPYING included with this distribution for more information. */ -#ifndef _DATA_FILE_READER_FACTORY_H_ -#define _DATA_FILE_READER_FACTORY_H_ +#ifndef SV_DATA_FILE_READER_FACTORY_H +#define SV_DATA_FILE_READER_FACTORY_H #include <QString> @@ -23,6 +23,7 @@ class DataFileReader; class Model; +class ProgressReporter; class DataFileReaderFactory { @@ -48,7 +49,8 @@ */ static DataFileReader *createReader(QString path, MIDIFileImportPreferenceAcquirer *, - sv_samplerate_t mainModelSampleRate); + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); /** * Read the given path, if a suitable reader is available. @@ -60,7 +62,8 @@ */ static Model *load(QString path, MIDIFileImportPreferenceAcquirer *acquirer, - sv_samplerate_t mainModelSampleRate); + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); /** * Read the given path, if a suitable reader is available. @@ -69,7 +72,8 @@ */ static Model *loadNonCSV(QString path, MIDIFileImportPreferenceAcquirer *acquirer, - sv_samplerate_t mainModelSampleRate); + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); /** * Read the given path using the CSV reader with the given format. @@ -77,13 +81,15 @@ */ static Model *loadCSV(QString path, CSVFormat format, - sv_samplerate_t mainModelSampleRate); + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); protected: static DataFileReader *createReader(QString path, bool csv, MIDIFileImportPreferenceAcquirer *, CSVFormat format, - sv_samplerate_t mainModelSampleRate); + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); }; #endif
--- a/data/fileio/MIDIFileReader.cpp Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/MIDIFileReader.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -58,7 +58,8 @@ MIDIFileReader::MIDIFileReader(QString path, MIDIFileImportPreferenceAcquirer *acquirer, - sv_samplerate_t mainModelSampleRate) : + sv_samplerate_t mainModelSampleRate, + ProgressReporter *) : // we don't actually report progress m_smpte(false), m_timingDivision(0), m_fps(0),
--- a/data/fileio/MIDIFileReader.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/MIDIFileReader.h Wed Sep 12 15:57:49 2018 +0100 @@ -31,6 +31,7 @@ #include <QObject> class MIDIEvent; +class ProgressReporter; typedef unsigned char MIDIByte; @@ -61,7 +62,8 @@ public: MIDIFileReader(QString path, MIDIFileImportPreferenceAcquirer *pref, // may be null - sv_samplerate_t mainModelSampleRate); + sv_samplerate_t mainModelSampleRate, + ProgressReporter *reporter = 0); virtual ~MIDIFileReader(); virtual bool isOK() const;
--- a/data/fileio/WavFileReader.cpp Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/WavFileReader.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -25,13 +25,17 @@ using namespace std; -WavFileReader::WavFileReader(FileSource source, bool fileUpdating) : +WavFileReader::WavFileReader(FileSource source, + bool fileUpdating, + Normalisation normalisation) : m_file(0), m_source(source), m_path(source.getLocalFilename()), m_seekable(false), m_lastStart(0), m_lastCount(0), + m_normalisation(normalisation), + m_max(0.f), m_updating(fileUpdating) { m_frameCount = 0; @@ -87,9 +91,13 @@ // and mark those (basically only non-adaptive WAVs). m_seekable = true; } + + if (m_normalisation != Normalisation::None && !m_updating) { + m_max = getMax(); + } } - SVDEBUG << "WavFileReader: Filename " << m_path << ", frame count " << m_frameCount << ", channel count " << m_channelCount << ", sample rate " << m_sampleRate << ", format " << m_fileInfo.format << ", seekable " << m_fileInfo.seekable << " adjusted to " << m_seekable << endl; + SVDEBUG << "WavFileReader: Filename " << m_path << ", frame count " << m_frameCount << ", channel count " << m_channelCount << ", sample rate " << m_sampleRate << ", format " << m_fileInfo.format << ", seekable " << m_fileInfo.seekable << " adjusted to " << m_seekable << ", normalisation " << int(m_normalisation) << endl; } WavFileReader::~WavFileReader() @@ -136,11 +144,31 @@ { updateFrameCount(); m_updating = false; + if (m_normalisation != Normalisation::None) { + m_max = getMax(); + } } floatvec_t WavFileReader::getInterleavedFrames(sv_frame_t start, sv_frame_t count) const { + floatvec_t frames = getInterleavedFramesUnnormalised(start, count); + + if (m_normalisation == Normalisation::None || m_max == 0.f) { + return frames; + } + + for (int i = 0; in_range_for(frames, i); ++i) { + frames[i] /= m_max; + } + + return frames; +} + +floatvec_t +WavFileReader::getInterleavedFramesUnnormalised(sv_frame_t start, + sv_frame_t count) const +{ static HitCount lastRead("WavFileReader: last read"); if (count == 0) return {}; @@ -200,6 +228,43 @@ return data; } +float +WavFileReader::getMax() const +{ + if (!m_file || !m_channelCount) { + return 0.f; + } + + // First try for a PEAK chunk + + double sfpeak = 0.0; + if (sf_command(m_file, SFC_GET_SIGNAL_MAX, &sfpeak, sizeof(sfpeak)) + == SF_TRUE) { + SVDEBUG << "File has a PEAK chunk reporting max level " << sfpeak + << endl; + return float(fabs(sfpeak)); + } + + // Failing that, read all the samples + + float peak = 0.f; + sv_frame_t ix = 0, chunk = 65536; + + while (ix < m_frameCount) { + auto frames = getInterleavedFrames(ix, chunk); + for (float x: frames) { + float level = fabsf(x); + if (level > peak) { + peak = level; + } + } + ix += chunk; + } + + SVDEBUG << "Measured file peak max level as " << peak << endl; + return peak; +} + void WavFileReader::getSupportedExtensions(set<QString> &extensions) {
--- a/data/fileio/WavFileReader.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/WavFileReader.h Wed Sep 12 15:57:49 2018 +0100 @@ -41,7 +41,11 @@ class WavFileReader : public AudioFileReader { public: - WavFileReader(FileSource source, bool fileUpdating = false); + enum class Normalisation { None, Peak }; + + WavFileReader(FileSource source, + bool fileUpdating = false, + Normalisation normalise = Normalisation::None); virtual ~WavFileReader(); virtual QString getLocation() const { return m_source.getLocation(); } @@ -55,7 +59,8 @@ * Must be safe to call from multiple threads with different * arguments on the same object at the same time. */ - virtual floatvec_t getInterleavedFrames(sv_frame_t start, sv_frame_t count) const; + virtual floatvec_t getInterleavedFrames(sv_frame_t start, sv_frame_t count) + const; static void getSupportedExtensions(std::set<QString> &extensions); static bool supportsExtension(QString ext); @@ -84,7 +89,14 @@ mutable sv_frame_t m_lastStart; mutable sv_frame_t m_lastCount; + Normalisation m_normalisation; + float m_max; + bool m_updating; + + floatvec_t getInterleavedFramesUnnormalised(sv_frame_t start, + sv_frame_t count) const; + float getMax() const; }; #endif
--- a/data/fileio/WavFileWriter.cpp Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/WavFileWriter.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -21,6 +21,9 @@ #include "base/Exceptions.h" #include "base/Debug.h" +#include <bqvec/Allocators.h> +#include <bqvec/VectorOps.h> + #include <QFileInfo> #include <iostream> @@ -63,10 +66,15 @@ m_file = sf_open(writePath.toLocal8Bit(), SFM_WRITE, &fileInfo); #endif if (!m_file) { - SVCERR << "WavFileWriter: Failed to open file (" - << sf_strerror(m_file) << ")" << endl; + SVCERR << "WavFileWriter: Failed to create float-WAV file of " + << m_channels << " channels at rate " << fileRate << " (" + << sf_strerror(m_file) << ")" << endl; m_error = QString("Failed to open audio file '%1' for writing") .arg(writePath); + if (m_temp) { + delete m_temp; + m_temp = 0; + } } } catch (FileOperationFailed &f) { m_error = f.what(); @@ -191,7 +199,18 @@ return isOK(); } - + +bool +WavFileWriter::putInterleavedFrames(const floatvec_t &frames) +{ + sv_frame_t count = frames.size() / m_channels; + float **samples = breakfastquay::allocate_channels<float>(m_channels, count); + breakfastquay::v_deinterleave(samples, frames.data(), m_channels, count); + bool result = writeSamples(samples, count); + breakfastquay::deallocate_channels(samples, m_channels); + return result; +} + bool WavFileWriter::close() {
--- a/data/fileio/WavFileWriter.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/fileio/WavFileWriter.h Wed Sep 12 15:57:49 2018 +0100 @@ -64,7 +64,11 @@ bool writeModel(DenseTimeValueModel *source, MultiSelection *selection = 0); - bool writeSamples(const float *const *samples, sv_frame_t count); // count per channel + /// Write samples from raw arrays; count is per-channel + bool writeSamples(const float *const *samples, sv_frame_t count); + + /// As writeSamples, but compatible with WavFileReader api. More expensive. + bool putInterleavedFrames(const floatvec_t &frames); bool close();
--- a/data/model/ReadOnlyWaveFileModel.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/model/ReadOnlyWaveFileModel.h Wed Sep 12 15:57:49 2018 +0100 @@ -36,8 +36,20 @@ Q_OBJECT public: + /** + * Construct a WaveFileModel from a source path and optional + * resampling target rate + */ ReadOnlyWaveFileModel(FileSource source, sv_samplerate_t targetRate = 0); + + /** + * Construct a WaveFileModel from a source path using an existing + * AudioFileReader. The model does not take ownership of the + * AudioFileReader, which remains managed by the caller and must + * outlive the model. + */ ReadOnlyWaveFileModel(FileSource source, AudioFileReader *reader); + ~ReadOnlyWaveFileModel(); bool isOK() const;
--- a/data/model/WritableWaveFileModel.cpp Tue Sep 04 11:31:35 2018 +0100 +++ b/data/model/WritableWaveFileModel.cpp Wed Sep 12 15:57:49 2018 +0100 @@ -36,47 +36,113 @@ //#define DEBUG_WRITABLE_WAVE_FILE_MODEL 1 -WritableWaveFileModel::WritableWaveFileModel(sv_samplerate_t sampleRate, +WritableWaveFileModel::WritableWaveFileModel(QString path, + sv_samplerate_t sampleRate, int channels, - QString path) : + Normalisation norm) : m_model(0), - m_writer(0), + m_temporaryWriter(0), + m_targetWriter(0), m_reader(0), + m_normalisation(norm), m_sampleRate(sampleRate), m_channels(channels), m_frameCount(0), m_startFrame(0), m_proportion(PROPORTION_UNKNOWN) { + init(path); +} + +WritableWaveFileModel::WritableWaveFileModel(sv_samplerate_t sampleRate, + int channels, + Normalisation norm) : + m_model(0), + m_temporaryWriter(0), + m_targetWriter(0), + m_reader(0), + m_normalisation(norm), + m_sampleRate(sampleRate), + m_channels(channels), + m_frameCount(0), + m_startFrame(0), + m_proportion(PROPORTION_UNKNOWN) +{ + init(); +} + +WritableWaveFileModel::WritableWaveFileModel(sv_samplerate_t sampleRate, + int channels) : + m_model(0), + m_temporaryWriter(0), + m_targetWriter(0), + m_reader(0), + m_normalisation(Normalisation::None), + m_sampleRate(sampleRate), + m_channels(channels), + m_frameCount(0), + m_startFrame(0), + m_proportion(PROPORTION_UNKNOWN) +{ + init(); +} + +void +WritableWaveFileModel::init(QString path) +{ if (path.isEmpty()) { try { + // Temp dir is exclusive to this run of the application, + // so the filename only needs to be unique within that - + // model ID should be ok QDir dir(TempDirectory::getInstance()->getPath()); - path = dir.filePath(QString("written_%1.wav") - .arg((intptr_t)this)); + path = dir.filePath(QString("written_%1.wav").arg(getId())); } catch (const DirectoryCreationFailed &f) { SVCERR << "WritableWaveFileModel: Failed to create temporary directory" << endl; return; } } - // Write directly to the target file, so that we can do - // incremental writes and concurrent reads - m_writer = new WavFileWriter(path, sampleRate, channels, - WavFileWriter::WriteToTarget); - if (!m_writer->isOK()) { - SVCERR << "WritableWaveFileModel: Error in creating WAV file writer: " << m_writer->getError() << endl; - delete m_writer; - m_writer = 0; + m_targetPath = path; + m_temporaryPath = ""; + + // We don't delete or null-out writer/reader members after + // failures here - they are all deleted in the dtor, and the + // presence/existence of the model is what's used to determine + // whether to go ahead, not the writer/readers. If the model is + // non-null, then the necessary writer/readers must be OK, as the + // model is the last thing initialised + + m_targetWriter = new WavFileWriter(m_targetPath, m_sampleRate, m_channels, + WavFileWriter::WriteToTarget); + + if (!m_targetWriter->isOK()) { + SVCERR << "WritableWaveFileModel: Error in creating WAV file writer: " << m_targetWriter->getError() << endl; return; } + + if (m_normalisation != Normalisation::None) { - FileSource source(m_writer->getPath()); + // Temp dir is exclusive to this run of the application, so + // the filename only needs to be unique within that + QDir dir(TempDirectory::getInstance()->getPath()); + m_temporaryPath = dir.filePath(QString("prenorm_%1.wav").arg(getId())); + + m_temporaryWriter = new WavFileWriter + (m_temporaryPath, m_sampleRate, m_channels, + WavFileWriter::WriteToTarget); + + if (!m_temporaryWriter->isOK()) { + SVCERR << "WritableWaveFileModel: Error in creating temporary WAV file writer: " << m_temporaryWriter->getError() << endl; + return; + } + } + + FileSource source(m_targetPath); m_reader = new WavFileReader(source, true); if (!m_reader->getError().isEmpty()) { - SVCERR << "WritableWaveFileModel: Error in creating wave file reader" << endl; - delete m_reader; - m_reader = 0; + SVCERR << "WritableWaveFileModel: Error in creating wave file reader: " << m_reader->getError() << endl; return; } @@ -85,8 +151,6 @@ SVCERR << "WritableWaveFileModel: Error in creating wave file model" << endl; delete m_model; m_model = 0; - delete m_reader; - m_reader = 0; return; } m_model->setStartFrame(m_startFrame); @@ -99,7 +163,8 @@ WritableWaveFileModel::~WritableWaveFileModel() { delete m_model; - delete m_writer; + delete m_targetWriter; + delete m_temporaryWriter; delete m_reader; } @@ -107,27 +172,36 @@ WritableWaveFileModel::setStartFrame(sv_frame_t startFrame) { m_startFrame = startFrame; - if (m_model) m_model->setStartFrame(startFrame); + if (m_model) { + m_model->setStartFrame(startFrame); + } } bool WritableWaveFileModel::addSamples(const float *const *samples, sv_frame_t count) { - if (!m_writer) return false; + if (!m_model) return false; #ifdef DEBUG_WRITABLE_WAVE_FILE_MODEL // SVDEBUG << "WritableWaveFileModel::addSamples(" << count << ")" << endl; #endif - if (!m_writer->writeSamples(samples, count)) { - SVCERR << "ERROR: WritableWaveFileModel::addSamples: writer failed: " << m_writer->getError() << endl; + WavFileWriter *writer = m_targetWriter; + if (m_normalisation != Normalisation::None) { + writer = m_temporaryWriter; + } + + if (!writer->writeSamples(samples, count)) { + SVCERR << "ERROR: WritableWaveFileModel::addSamples: writer failed: " << writer->getError() << endl; return false; } m_frameCount += count; - if (m_reader && m_reader->getChannelCount() == 0) { - m_reader->updateFrameCount(); + if (m_normalisation == Normalisation::None) { + if (m_reader->getChannelCount() == 0) { + m_reader->updateFrameCount(); + } } return true; @@ -136,17 +210,15 @@ void WritableWaveFileModel::updateModel() { - if (m_reader) { - m_reader->updateFrameCount(); - } + if (!m_model) return; + + m_reader->updateFrameCount(); } bool WritableWaveFileModel::isOK() const { - bool ok = (m_writer && m_writer->isOK()); -// SVDEBUG << "WritableWaveFileModel::isOK(): ok = " << ok << endl; - return ok; + return (m_model && m_model->isOK()); } bool @@ -173,12 +245,56 @@ void WritableWaveFileModel::writeComplete() { - m_writer->close(); - if (m_reader) m_reader->updateDone(); + if (!m_model) return; + + if (m_normalisation == Normalisation::None) { + m_targetWriter->close(); + } else { + m_temporaryWriter->close(); + normaliseToTarget(); + } + + m_reader->updateDone(); m_proportion = 100; emit modelChanged(); } +void +WritableWaveFileModel::normaliseToTarget() +{ + if (m_temporaryPath == "") { + SVCERR << "WritableWaveFileModel::normaliseToTarget: No temporary path available" << endl; + return; + } + + WavFileReader normalisingReader(m_temporaryPath, false, + WavFileReader::Normalisation::Peak); + + if (!normalisingReader.getError().isEmpty()) { + SVCERR << "WritableWaveFileModel: Error in creating normalising reader: " << normalisingReader.getError() << endl; + return; + } + + sv_frame_t frame = 0; + sv_frame_t block = 65536; + sv_frame_t count = normalisingReader.getFrameCount(); + + while (frame < count) { + auto frames = normalisingReader.getInterleavedFrames(frame, block); + if (!m_targetWriter->putInterleavedFrames(frames)) { + SVCERR << "ERROR: WritableWaveFileModel::normaliseToTarget: writer failed: " << m_targetWriter->getError() << endl; + return; + } + frame += block; + } + + m_targetWriter->close(); + + delete m_temporaryWriter; + m_temporaryWriter = 0; + QFile::remove(m_temporaryPath); +} + sv_frame_t WritableWaveFileModel::getFrameCount() const { @@ -239,7 +355,7 @@ Model::toXml (out, indent, QString("type=\"wavefile\" file=\"%1\" subtype=\"writable\" %2") - .arg(encodeEntities(m_writer->getPath())) + .arg(encodeEntities(m_targetPath)) .arg(extraAttributes)); }
--- a/data/model/WritableWaveFileModel.h Tue Sep 04 11:31:35 2018 +0100 +++ b/data/model/WritableWaveFileModel.h Wed Sep 12 15:57:49 2018 +0100 @@ -28,7 +28,54 @@ Q_OBJECT public: - WritableWaveFileModel(sv_samplerate_t sampleRate, int channels, QString path = ""); + enum class Normalisation { None, Peak }; + + /** + * Create a WritableWaveFileModel of the given sample rate and + * channel count, storing data in a new float-type extended WAV + * file with the given path. If path is the empty string, the data + * will be stored in a newly-created temporary file. + * + * If normalisation == None, sample values will be written + * verbatim, and will be ready to read as soon as they have been + * written. Otherwise samples will be normalised on writing; this + * will require an additional pass and temporary file, and no + * samples will be available to read until after writeComplete() + * has returned. + */ + WritableWaveFileModel(QString path, + sv_samplerate_t sampleRate, + int channels, + Normalisation normalisation); + + /** + * Create a WritableWaveFileModel of the given sample rate and + * channel count, storing data in a new float-type extended WAV + * file in a temporary location. This is equivalent to passing an + * empty path to the constructor above. + * + * If normalisation == None, sample values will be written + * verbatim, and will be ready to read as soon as they have been + * written. Otherwise samples will be normalised on writing; this + * will require an additional pass and temporary file, and no + * samples will be available to read until after writeComplete() + * has returned. + */ + WritableWaveFileModel(sv_samplerate_t sampleRate, + int channels, + Normalisation normalisation); + + /** + * Create a WritableWaveFileModel of the given sample rate and + * channel count, storing data in a new float-type extended WAV + * file in a temporary location, and applying no normalisation. + * + * This is equivalent to passing an empty path and + * Normalisation::None to the first constructor above. + */ + WritableWaveFileModel(sv_samplerate_t sampleRate, + int channels); + ~WritableWaveFileModel(); /** @@ -152,13 +199,35 @@ protected: ReadOnlyWaveFileModel *m_model; - WavFileWriter *m_writer; + + /** When normalising, this writer is used to write verbatim + * samples to the temporary file prior to + * normalisation. Otherwise it's null + */ + WavFileWriter *m_temporaryWriter; + QString m_temporaryPath; + + /** When not normalising, this writer is used to write verbatim + * samples direct to the target file. When normalising, it is + * used to write normalised samples to the target after the + * temporary file has been completed. But it is still created on + * initialisation, so that there is a file header ready for the + * reader to address. + */ + WavFileWriter *m_targetWriter; + QString m_targetPath; + WavFileReader *m_reader; + Normalisation m_normalisation; sv_samplerate_t m_sampleRate; int m_channels; sv_frame_t m_frameCount; sv_frame_t m_startFrame; int m_proportion; + +private: + void init(QString path = ""); + void normaliseToTarget(); }; #endif
--- a/files.pri Tue Sep 04 11:31:35 2018 +0100 +++ b/files.pri Wed Sep 12 15:57:49 2018 +0100 @@ -24,6 +24,7 @@ base/RangeMapper.h \ base/RealTime.h \ base/RecentFiles.h \ + base/RecordDirectory.h \ base/ResourceFinder.h \ base/RingBuffer.h \ base/ScaleTickIntervals.h \ @@ -160,6 +161,7 @@ base/RangeMapper.cpp \ base/RealTimeSV.cpp \ base/RecentFiles.cpp \ + base/RecordDirectory.cpp \ base/ResourceFinder.cpp \ base/Selection.cpp \ base/Serialiser.cpp \