changeset 1434:0684c6698e3f streaming-csv-writer

Added utility function for splitting a model selection into chunks and writing to a stream.

--- a/data/fileio/CSVStreamWriter.h	Tue Apr 17 10:03:49 2018 +0100
+++ b/data/fileio/CSVStreamWriter.h	Tue Apr 17 10:03:49 2018 +0100
@@ -16,11 +16,85 @@
 #ifndef _CSV_STREAM_WRITER_H_
 #define _CSV_STREAM_WRITER_H_
 
-class CSVStreamWriter
+#include "base/BaseTypes.h"
+#include "base/Selection.h"
+#include "base/ProgressReporter.h"
+#include "base/DataExportOptions.h"
+#include "data/model/Model.h"
+#include <QString>
+#include <algorithm>
+
+namespace CSV
 {
+using Completed = bool;
 
-};
+template <class OutStream>
+auto writeToStreamInChunks(
+    OutStream& oss,
+    const Model& model,
+    const Selection& extents,
+    ProgressReporter* reporter = nullptr,
+    QString delimiter = ",",
+    DataExportOptions options = DataExportDefaults,
+    const sv_frame_t blockSize = 16384
+) -> Completed
+{
+    if (blockSize <= 0) return false;
+    sv_frame_t readPtr = extents.isEmpty() ?
+        model.getStartFrame() : extents.getStartFrame();
+    sv_frame_t endFrame = extents.isEmpty() ?
+        model.getEndFrame() : extents.getEndFrame();
+    int previousPercentagePoint = 0;
 
-#endif
+    const auto wasCancelled = [&reporter]() { 
+        return reporter && reporter->wasCancelled(); 
+    };
 
-    
+    while (readPtr < endFrame) {
+        if (wasCancelled()) return false;
+
+        const auto start = readPtr;
+        const auto end = std::min(start + blockSize, endFrame);
+
+        oss << model.toDelimitedDataStringSubsetWithOptions(
+            delimiter,
+            options,
+            start,
+            end
+        ) << (end < endFrame ? "\n" : "");
+
+        const auto currentPercentage = 100 * end / endFrame;
+        const bool hasIncreased = currentPercentage > previousPercentagePoint;
+
+        if (hasIncreased) {
+            if (reporter) reporter->setProgress(currentPercentage);
+            previousPercentagePoint = currentPercentage;
+        }
+        readPtr = end;
+    }
+    return !wasCancelled(); // setProgress could process event loop
+}
+
+template <class OutStream>
+auto writeToStreamInChunks(
+    OutStream& oss,
+    const Model& model,
+    ProgressReporter* reporter = nullptr,
+    QString delimiter = ",",
+    DataExportOptions options = DataExportDefaults,
+    const sv_frame_t blockSize = 16384
+) -> Completed
+{
+    const Selection empty;
+    return CSV::writeToStreamInChunks(
+        oss,
+        model,
+        empty,
+        reporter,
+        delimiter,
+        options,
+        blockSize
+    );
+}
+} // namespace
+#endif
\ No newline at end of file

--- a/data/fileio/test/CSVStreamWriterTest.h	Tue Apr 17 10:03:49 2018 +0100
+++ b/data/fileio/test/CSVStreamWriterTest.h	Tue Apr 17 10:03:49 2018 +0100
@@ -4,7 +4,7 @@
     Sonic Visualiser
     An audio file viewer and annotation editor.
     Centre for Digital Music, Queen Mary, University of London.
-    This file copyright 2013 Chris Cannam.
+    This file copyright 2017 Lucas Thompson.
     
     This program is free software; you can redistribute it and/or
     modify it under the terms of the GNU General Public License as
@@ -16,18 +16,158 @@
 #ifndef TEST_CSV_STREAM_H
 #define TEST_CSV_STREAM_H
 
+#include <QtTest>
 #include <QObject>
+#include <sstream>
+#include <functional>
+
+#include "base/ProgressReporter.h"
+#include "base/DataExportOptions.h"
 #include "../CSVStreamWriter.h"
+#include "../../model/test/MockWaveModel.h"
+
+namespace
+{
+    class StubReporter : public ProgressReporter
+    {
+    public:
+        StubReporter( std::function<bool()> isCancelled )
+            : m_isCancelled(isCancelled) {}
+        bool isDefinite() const override { return true; }
+        void setDefinite(bool) override {}
+        bool wasCancelled() const override { return m_isCancelled(); }
+        void setMessage(QString) override {}
+        void setProgress(int) override { ++m_calls; }
+        size_t getCallCount() const { return m_calls; }
+        void reset() { m_calls = 0; }
+    private:
+        size_t m_calls = 0;
+        std::function<bool()> m_isCancelled;
+    };
+} // namespace
 
 class CSVStreamWriterTest : public QObject
 {
     Q_OBJECT
 public:
+    std::string getExpectedString()
+    {
+        return
+        {
+          "0,0,0\n"
+          "1,0,0\n"
+          "2,0,0\n"
+          "3,0,0\n"
+          "4,1,1\n"
+          "5,1,1\n"
+          "6,1,1\n"
+          "7,1,1\n"
+          "8,1,1\n"
+          "9,1,1\n"
+          "10,1,1\n"
+          "11,1,1\n"
+          "12,1,1\n"
+          "13,1,1\n"
+          "14,1,1\n"
+          "15,1,1\n"
+          "16,1,1\n"
+          "17,1,1\n"
+          "18,1,1\n"
+          "19,1,1\n"
+          "20,0,0\n"
+          "21,0,0\n"
+          "22,0,0\n"
+          "23,0,0"
+        };
+    }
 
 private slots:
-    void toDo()
+    void simpleValidOutput()
     {
+        MockWaveModel mwm({ DC, DC }, 16, 4);
 
+        std::ostringstream oss;
+        const auto result = CSV::writeToStreamInChunks(oss, mwm);
+        QVERIFY( oss.str() == getExpectedString() );
+        QVERIFY( result );
+    }
+
+    void callsReporterCorrectTimes()
+    {
+        MockWaveModel mwm({ DC, DC }, 16, 4);
+        StubReporter reporter { []() -> bool { return false; } };
+        const auto expected = getExpectedString();
+
+        std::ostringstream oss;
+        const auto writeStreamWithBlockSize = [&](int blockSize) {
+            return CSV::writeToStreamInChunks(
+                oss,
+                mwm,
+                &reporter,
+                ",",
+                DataExportDefaults,
+                blockSize
+            );
+        };
+
+        const auto reset = [&]() {
+            oss.str({});
+            reporter.reset();
+        };
+
+        const auto nonIntegerMultipleResult = writeStreamWithBlockSize(5);
+        QVERIFY( nonIntegerMultipleResult );
+        QVERIFY( reporter.getCallCount() == 5 /* 4.8 rounded up */ );
+        QVERIFY( oss.str() == expected );
+        reset();
+
+        const auto integerMultiple = writeStreamWithBlockSize(2);
+        QVERIFY( integerMultiple );
+        QVERIFY( reporter.getCallCount() == 12 );
+        QVERIFY( oss.str() == expected );
+        reset();
+
+        const auto largerThanNumberOfSamples = writeStreamWithBlockSize(100);
+        QVERIFY( largerThanNumberOfSamples );
+        QVERIFY( reporter.getCallCount() == 1 );
+        QVERIFY( oss.str() == expected );
+        reset();
+
+        const auto zero = writeStreamWithBlockSize(0);
+        QVERIFY( zero == false );
+        QVERIFY( reporter.getCallCount() == 0 );
+    }
+
+    void isCancellable()
+    {
+        MockWaveModel mwm({ DC, DC }, 16, 4);
+        StubReporter reporter { []() -> bool { return true; } };
+
+        std::ostringstream oss;
+        const auto cancelImmediately = CSV::writeToStreamInChunks(
+            oss,
+            mwm,
+            &reporter,
+            ",",
+            DataExportDefaults,
+            4
+        );
+        QVERIFY( cancelImmediately == false );
+        QVERIFY( reporter.getCallCount() == 0 );
+
+        StubReporter cancelMidway { 
+            [&]() { return cancelMidway.getCallCount() == 3; } 
+        };
+        const auto cancelledMidway = CSV::writeToStreamInChunks(
+            oss,
+            mwm,
+            &cancelMidway,
+            ",",
+            DataExportDefaults,
+            4
+        );
+        QVERIFY( cancelMidway.getCallCount() == 3 );
+        QVERIFY( cancelledMidway == false );
     }
 };
 

--- a/data/fileio/test/files.pri	Tue Apr 17 10:03:49 2018 +0100
+++ b/data/fileio/test/files.pri	Tue Apr 17 10:03:49 2018 +0100
@@ -1,6 +1,7 @@
 
 TEST_HEADERS += \
 	../../../test/TestHelper.h \
+	../../model/test/MockWaveModel.h \
 	AudioFileReaderTest.h \
 	AudioFileWriterTest.h \
 	AudioTestData.h \
@@ -9,4 +10,5 @@
 	CSVStreamWriterTest.h
      
 TEST_SOURCES += \
+	../../model/test/MockWaveModel.cpp \
 	svcore-data-fileio-test.cpp

--- a/files.pri	Tue Apr 17 10:03:49 2018 +0100
+++ b/files.pri	Tue Apr 17 10:03:49 2018 +0100
@@ -181,7 +181,6 @@
            data/fileio/CSVFileReader.cpp \
            data/fileio/CSVFileWriter.cpp \
            data/fileio/CSVFormat.cpp \
-           data/fileio/CSVStreamWriter.cpp \
            data/fileio/DataFileReaderFactory.cpp \
            data/fileio/FileReadThread.cpp \
            data/fileio/FileSource.cpp \