changeset 1520:954d0cf29ca7 import-audio-data

Switch the normalisation option in WritableWaveFileModel from normalising on read to normalising on write, so that the saved file is already normalised and therefore can be read again without having to remember to normalise it

--- a/data/fileio/CSVFileReader.cpp	Tue Sep 11 14:36:51 2018 +0100
+++ b/data/fileio/CSVFileReader.cpp	Wed Sep 12 13:56:56 2018 +0100
@@ -348,7 +348,10 @@
                                       == CSVFormat::SampleRangeOther);
                     QString path = getConvertedAudioFilePath();
                     modelW = new WritableWaveFileModel
-                        (sampleRate, valueColumns, path, normalise);
+                        (path, sampleRate, valueColumns,
+                         normalise ?
+                         WritableWaveFileModel::Normalisation::Peak :
+                         WritableWaveFileModel::Normalisation::None);
                     modelName = QFileInfo(path).fileName();
                     model = modelW;
                     break;

--- a/data/fileio/WavFileReader.cpp	Tue Sep 11 14:36:51 2018 +0100
+++ b/data/fileio/WavFileReader.cpp	Wed Sep 12 13:56:56 2018 +0100
@@ -27,14 +27,14 @@
 
 WavFileReader::WavFileReader(FileSource source,
                              bool fileUpdating,
-                             bool normalise) :
+                             Normalisation normalisation) :
     m_file(0),
     m_source(source),
     m_path(source.getLocalFilename()),
     m_seekable(false),
     m_lastStart(0),
     m_lastCount(0),
-    m_normalise(normalise),
+    m_normalisation(normalisation),
     m_max(0.f),
     m_updating(fileUpdating)
 {
@@ -92,12 +92,12 @@
             m_seekable = true;
         }
 
-        if (m_normalise && !m_updating) {
+        if (m_normalisation != Normalisation::None && !m_updating) {
             m_max = getMax();
         }
     }
 
-    SVDEBUG << "WavFileReader: Filename " << m_path << ", frame count " << m_frameCount << ", channel count " << m_channelCount << ", sample rate " << m_sampleRate << ", format " << m_fileInfo.format << ", seekable " << m_fileInfo.seekable << " adjusted to " << m_seekable << ", normalise " << m_normalise << endl;
+    SVDEBUG << "WavFileReader: Filename " << m_path << ", frame count " << m_frameCount << ", channel count " << m_channelCount << ", sample rate " << m_sampleRate << ", format " << m_fileInfo.format << ", seekable " << m_fileInfo.seekable << " adjusted to " << m_seekable << ", normalisation " << int(m_normalisation) << endl;
 }
 
 WavFileReader::~WavFileReader()
@@ -144,7 +144,7 @@
 {
     updateFrameCount();
     m_updating = false;
-    if (m_normalise) {
+    if (m_normalisation != Normalisation::None) {
         m_max = getMax();
     }
 }
@@ -154,7 +154,7 @@
 {
     floatvec_t frames = getInterleavedFramesUnnormalised(start, count);
 
-    if (!m_normalise || m_max == 0.f) {
+    if (m_normalisation == Normalisation::None || m_max == 0.f) {
         return frames;
     }
 

--- a/data/fileio/WavFileReader.h	Tue Sep 11 14:36:51 2018 +0100
+++ b/data/fileio/WavFileReader.h	Wed Sep 12 13:56:56 2018 +0100
@@ -41,9 +41,11 @@
 class WavFileReader : public AudioFileReader
 {
 public:
+    enum class Normalisation { None, Peak };
+
     WavFileReader(FileSource source,
                   bool fileUpdating = false,
-                  bool normalise = false);
+                  Normalisation normalise = Normalisation::None);
     virtual ~WavFileReader();
 
     virtual QString getLocation() const { return m_source.getLocation(); }
@@ -87,7 +89,7 @@
     mutable sv_frame_t m_lastStart;
     mutable sv_frame_t m_lastCount;
 
-    bool m_normalise;
+    Normalisation m_normalisation;
     float m_max;
 
     bool m_updating;

--- a/data/fileio/WavFileWriter.cpp	Tue Sep 11 14:36:51 2018 +0100
+++ b/data/fileio/WavFileWriter.cpp	Wed Sep 12 13:56:56 2018 +0100
@@ -21,6 +21,9 @@
 #include "base/Exceptions.h"
 #include "base/Debug.h"
 
+#include <bqvec/Allocators.h>
+#include <bqvec/VectorOps.h>
+
 #include <QFileInfo>
 
 #include <iostream>
@@ -196,7 +199,18 @@
 
     return isOK();
 }
-    
+
+bool
+WavFileWriter::putInterleavedFrames(const floatvec_t &frames)
+{
+    sv_frame_t count = frames.size() / m_channels;
+    float **samples = breakfastquay::allocate_channels<float>(m_channels, count);
+    breakfastquay::v_deinterleave(samples, frames.data(), m_channels, count);
+    bool result = writeSamples(samples, count);
+    breakfastquay::deallocate_channels(samples, m_channels);
+    return result;
+}
+
 bool
 WavFileWriter::close()
 {

--- a/data/fileio/WavFileWriter.h	Tue Sep 11 14:36:51 2018 +0100
+++ b/data/fileio/WavFileWriter.h	Wed Sep 12 13:56:56 2018 +0100
@@ -64,7 +64,11 @@
     bool writeModel(DenseTimeValueModel *source,
                     MultiSelection *selection = 0);
 
-    bool writeSamples(const float *const *samples, sv_frame_t count); // count per channel
+    /// Write samples from raw arrays; count is per-channel
+    bool writeSamples(const float *const *samples, sv_frame_t count);
+
+    /// As writeSamples, but compatible with WavFileReader api. More expensive.
+    bool putInterleavedFrames(const floatvec_t &frames);
 
     bool close();
 

--- a/data/model/WritableWaveFileModel.cpp	Tue Sep 11 14:36:51 2018 +0100
+++ b/data/model/WritableWaveFileModel.cpp	Wed Sep 12 13:56:56 2018 +0100
@@ -36,48 +36,113 @@
 
 //#define DEBUG_WRITABLE_WAVE_FILE_MODEL 1
 
-WritableWaveFileModel::WritableWaveFileModel(sv_samplerate_t sampleRate,
+WritableWaveFileModel::WritableWaveFileModel(QString path,
+                                             sv_samplerate_t sampleRate,
                                              int channels,
-                                             QString path,
-                                             bool normaliseOnRead) :
+                                             Normalisation norm) :
     m_model(0),
-    m_writer(0),
+    m_temporaryWriter(0),
+    m_targetWriter(0),
     m_reader(0),
+    m_normalisation(norm),
     m_sampleRate(sampleRate),
     m_channels(channels),
     m_frameCount(0),
     m_startFrame(0),
     m_proportion(PROPORTION_UNKNOWN)
 {
+    init(path);
+}
+
+WritableWaveFileModel::WritableWaveFileModel(sv_samplerate_t sampleRate,
+                                             int channels,
+                                             Normalisation norm) :
+    m_model(0),
+    m_temporaryWriter(0),
+    m_targetWriter(0),
+    m_reader(0),
+    m_normalisation(norm),
+    m_sampleRate(sampleRate),
+    m_channels(channels),
+    m_frameCount(0),
+    m_startFrame(0),
+    m_proportion(PROPORTION_UNKNOWN)
+{
+    init();
+}
+
+WritableWaveFileModel::WritableWaveFileModel(sv_samplerate_t sampleRate,
+                                             int channels) :
+    m_model(0),
+    m_temporaryWriter(0),
+    m_targetWriter(0),
+    m_reader(0),
+    m_normalisation(Normalisation::None),
+    m_sampleRate(sampleRate),
+    m_channels(channels),
+    m_frameCount(0),
+    m_startFrame(0),
+    m_proportion(PROPORTION_UNKNOWN)
+{
+    init();
+}
+
+void
+WritableWaveFileModel::init(QString path)
+{
     if (path.isEmpty()) {
         try {
+            // Temp dir is exclusive to this run of the application,
+            // so the filename only needs to be unique within that -
+            // model ID should be ok
             QDir dir(TempDirectory::getInstance()->getPath());
-            path = dir.filePath(QString("written_%1.wav")
-                                .arg((intptr_t)this));
+            path = dir.filePath(QString("written_%1.wav").arg(getId()));
         } catch (const DirectoryCreationFailed &f) {
             SVCERR << "WritableWaveFileModel: Failed to create temporary directory" << endl;
             return;
         }
     }
 
-    // Write directly to the target file, so that we can do
-    // incremental writes and concurrent reads
-    m_writer = new WavFileWriter(path, sampleRate, channels,
-                                 WavFileWriter::WriteToTarget);
-    if (!m_writer->isOK()) {
-        SVCERR << "WritableWaveFileModel: Error in creating WAV file writer: " << m_writer->getError() << endl;
-        delete m_writer; 
-        m_writer = 0;
+    m_targetPath = path;
+    m_temporaryPath = "";
+
+    // We don't delete or null-out writer/reader members after
+    // failures here - they are all deleted in the dtor, and the
+    // presence/existence of the model is what's used to determine
+    // whether to go ahead, not the writer/readers. If the model is
+    // non-null, then the necessary writer/readers must be OK, as the
+    // model is the last thing initialised
+    
+    m_targetWriter = new WavFileWriter(m_targetPath, m_sampleRate, m_channels,
+                                       WavFileWriter::WriteToTarget);
+    
+    if (!m_targetWriter->isOK()) {
+        SVCERR << "WritableWaveFileModel: Error in creating WAV file writer: " << m_targetWriter->getError() << endl;
         return;
     }
+    
+    if (m_normalisation != Normalisation::None) {
 
-    FileSource source(m_writer->getPath());
+        // Temp dir is exclusive to this run of the application, so
+        // the filename only needs to be unique within that
+        QDir dir(TempDirectory::getInstance()->getPath());
+        m_temporaryPath = dir.filePath(QString("prenorm_%1.wav").arg(getId()));
 
-    m_reader = new WavFileReader(source, true, normaliseOnRead);
+        m_temporaryWriter = new WavFileWriter
+            (m_temporaryPath, m_sampleRate, m_channels,
+             WavFileWriter::WriteToTarget);
+    
+        if (!m_temporaryWriter->isOK()) {
+            SVCERR << "WritableWaveFileModel: Error in creating temporary WAV file writer: " << m_temporaryWriter->getError() << endl;
+            return;
+        }
+    }        
+
+    FileSource source(m_targetPath);
+
+    m_reader = new WavFileReader(source, true);
     if (!m_reader->getError().isEmpty()) {
-        SVCERR << "WritableWaveFileModel: Error in creating wave file reader" << endl;
-        delete m_reader;
-        m_reader = 0;
+        SVCERR << "WritableWaveFileModel: Error in creating wave file reader: " << m_reader->getError() << endl;
         return;
     }
     
@@ -86,8 +151,6 @@
         SVCERR << "WritableWaveFileModel: Error in creating wave file model" << endl;
         delete m_model;
         m_model = 0;
-        delete m_reader;
-        m_reader = 0;
         return;
     }
     m_model->setStartFrame(m_startFrame);
@@ -100,7 +163,8 @@
 WritableWaveFileModel::~WritableWaveFileModel()
 {
     delete m_model;
-    delete m_writer;
+    delete m_targetWriter;
+    delete m_temporaryWriter;
     delete m_reader;
 }
 
@@ -108,27 +172,36 @@
 WritableWaveFileModel::setStartFrame(sv_frame_t startFrame)
 {
     m_startFrame = startFrame;
-    if (m_model) m_model->setStartFrame(startFrame);
+    if (m_model) {
+        m_model->setStartFrame(startFrame);
+    }
 }
 
 bool
 WritableWaveFileModel::addSamples(const float *const *samples, sv_frame_t count)
 {
-    if (!m_writer) return false;
+    if (!m_model) return false;
 
 #ifdef DEBUG_WRITABLE_WAVE_FILE_MODEL
 //    SVDEBUG << "WritableWaveFileModel::addSamples(" << count << ")" << endl;
 #endif
 
-    if (!m_writer->writeSamples(samples, count)) {
-        SVCERR << "ERROR: WritableWaveFileModel::addSamples: writer failed: " << m_writer->getError() << endl;
+    WavFileWriter *writer = m_targetWriter;
+    if (m_normalisation != Normalisation::None) {
+        writer = m_temporaryWriter;
+    }
+    
+    if (!writer->writeSamples(samples, count)) {
+        SVCERR << "ERROR: WritableWaveFileModel::addSamples: writer failed: " << writer->getError() << endl;
         return false;
     }
 
     m_frameCount += count;
 
-    if (m_reader && m_reader->getChannelCount() == 0) {
-        m_reader->updateFrameCount();
+    if (m_normalisation == Normalisation::None) {
+        if (m_reader->getChannelCount() == 0) {
+            m_reader->updateFrameCount();
+        }
     }
 
     return true;
@@ -137,17 +210,15 @@
 void
 WritableWaveFileModel::updateModel()
 {
-    if (m_reader) {
-        m_reader->updateFrameCount();
-    }
+    if (!m_model) return;
+    
+    m_reader->updateFrameCount();
 }
 
 bool
 WritableWaveFileModel::isOK() const
 {
-    bool ok = (m_writer && m_writer->isOK());
-//    SVDEBUG << "WritableWaveFileModel::isOK(): ok = " << ok << endl;
-    return ok;
+    return (m_model && m_model->isOK());
 }
 
 bool
@@ -174,12 +245,56 @@
 void
 WritableWaveFileModel::writeComplete()
 {
-    m_writer->close();
-    if (m_reader) m_reader->updateDone();
+    if (!m_model) return;
+
+    if (m_normalisation == Normalisation::None) {
+        m_targetWriter->close();
+    } else {
+        m_temporaryWriter->close();
+        normaliseToTarget();
+    }
+    
+    m_reader->updateDone();
     m_proportion = 100;
     emit modelChanged();
 }
 
+void
+WritableWaveFileModel::normaliseToTarget()
+{
+    if (m_temporaryPath == "") {
+        SVCERR << "WritableWaveFileModel::normaliseToTarget: No temporary path available" << endl;
+        return;
+    }
+    
+    WavFileReader normalisingReader(m_temporaryPath, false,
+                                    WavFileReader::Normalisation::Peak);
+
+    if (!normalisingReader.getError().isEmpty()) {
+        SVCERR << "WritableWaveFileModel: Error in creating normalising reader: " << normalisingReader.getError() << endl;
+        return;
+    }
+
+    sv_frame_t frame = 0;
+    sv_frame_t block = 65536;
+    sv_frame_t count = normalisingReader.getFrameCount();
+
+    while (frame < count) {
+        auto frames = normalisingReader.getInterleavedFrames(frame, block);
+        if (!m_targetWriter->putInterleavedFrames(frames)) {
+            SVCERR << "ERROR: WritableWaveFileModel::normaliseToTarget: writer failed: " << m_targetWriter->getError() << endl;
+            return;
+        }
+        frame += block;
+    }
+
+    m_targetWriter->close();
+
+    delete m_temporaryWriter;
+    m_temporaryWriter = 0;
+    QFile::remove(m_temporaryPath);
+}
+
 sv_frame_t
 WritableWaveFileModel::getFrameCount() const
 {
@@ -240,7 +355,7 @@
     Model::toXml
         (out, indent,
          QString("type=\"wavefile\" file=\"%1\" subtype=\"writable\" %2")
-         .arg(encodeEntities(m_writer->getPath()))
+         .arg(encodeEntities(m_targetPath))
          .arg(extraAttributes));
 }
 

--- a/data/model/WritableWaveFileModel.h	Tue Sep 11 14:36:51 2018 +0100
+++ b/data/model/WritableWaveFileModel.h	Wed Sep 12 13:56:56 2018 +0100
@@ -28,10 +28,54 @@
     Q_OBJECT
 
 public:
+    enum class Normalisation { None, Peak };
+
+    /**
+     * Create a WritableWaveFileModel of the given sample rate and
+     * channel count, storing data in a new float-type extended WAV
+     * file with the given path. If path is the empty string, the data
+     * will be stored in a newly-created temporary file.
+     *
+     * If normalisation == None, sample values will be written
+     * verbatim, and will be ready to read as soon as they have been
+     * written. Otherwise samples will be normalised on writing; this
+     * will require an additional pass and temporary file, and no
+     * samples will be available to read until after writeComplete()
+     * has returned.
+     */
+    WritableWaveFileModel(QString path,
+                          sv_samplerate_t sampleRate,
+                          int channels,
+                          Normalisation normalisation);
+    
+    /**
+     * Create a WritableWaveFileModel of the given sample rate and
+     * channel count, storing data in a new float-type extended WAV
+     * file in a temporary location. This is equivalent to passing an
+     * empty path to the constructor above.
+     *
+     * If normalisation == None, sample values will be written
+     * verbatim, and will be ready to read as soon as they have been
+     * written. Otherwise samples will be normalised on writing; this
+     * will require an additional pass and temporary file, and no
+     * samples will be available to read until after writeComplete()
+     * has returned.
+     */
     WritableWaveFileModel(sv_samplerate_t sampleRate,
                           int channels,
-                          QString path = "",
-                          bool normaliseOnRead = false);
+                          Normalisation normalisation);
+
+    /**
+     * Create a WritableWaveFileModel of the given sample rate and
+     * channel count, storing data in a new float-type extended WAV
+     * file in a temporary location, and applying no normalisation.
+     *
+     * This is equivalent to passing an empty path and
+     * Normalisation::None to the first constructor above.
+     */
+    WritableWaveFileModel(sv_samplerate_t sampleRate,
+                          int channels);
+
     ~WritableWaveFileModel();
 
     /**
@@ -155,13 +199,35 @@
 
 protected:
     ReadOnlyWaveFileModel *m_model;
-    WavFileWriter *m_writer;
+
+    /** When normalising, this writer is used to write verbatim
+     *  samples to the temporary file prior to
+     *  normalisation. Otherwise it's null
+     */
+    WavFileWriter *m_temporaryWriter;
+    QString m_temporaryPath;
+
+    /** When not normalising, this writer is used to write verbatim
+     *  samples direct to the target file. When normalising, it is
+     *  used to write normalised samples to the target after the
+     *  temporary file has been completed. But it is still created on
+     *  initialisation, so that there is a file header ready for the
+     *  reader to address.
+     */
+    WavFileWriter *m_targetWriter;
+    QString m_targetPath;
+
     WavFileReader *m_reader;
+    Normalisation m_normalisation;
     sv_samplerate_t m_sampleRate;
     int m_channels;
     sv_frame_t m_frameCount;
     sv_frame_t m_startFrame;
     int m_proportion;
+
+private:
+    void init(QString path = "");
+    void normaliseToTarget();
 };
 
 #endif