changeset 1445:4437209348ef streaming-csv-writer

Introduce overload of writeInChunks taking a MultiSelection. Simplifies the progress reporting.

--- a/data/fileio/CSVFileWriter.cpp	Tue Apr 17 10:03:50 2018 +0100
+++ b/data/fileio/CSVFileWriter.cpp	Tue Apr 17 10:03:50 2018 +0100
@@ -29,7 +29,6 @@
 #include <QFile>
 #include <QTextStream>
 #include <exception>
-#include <numeric>
 
 CSVFileWriter::CSVFileWriter(QString path,
                              Model *model,
@@ -86,52 +85,15 @@
     
         QTextStream out(&file);
 
-        bool completed = false;
-
-        const auto nFramesToWrite = std::accumulate(
-            selection->getSelections().begin(),
-            selection->getSelections().end(),
-            0,
-            [](sv_frame_t acc, const Selection& current) -> sv_frame_t {
-                return acc + (current.getEndFrame() - current.getStartFrame());
-            }
+        bool completed = CSVStreamWriter::writeInChunks(
+            out,
+            *m_model,
+            *selection,
+            m_reporter,
+            m_delimiter,
+            m_options
         );
 
-        sv_frame_t nFramesWritten = 0;
-        const auto createProgressCalculator = [
-            &nFramesWritten,
-            &nFramesToWrite
-        ](sv_frame_t nFramesToWriteForSelection) {
-            const auto nFramesWrittenAtSelectionStart = nFramesWritten;
-            nFramesWritten += nFramesToWriteForSelection;
-            return [
-                &nFramesWritten,
-                &nFramesToWrite,
-                nFramesWrittenAtSelectionStart
-            ] (sv_frame_t nFramesWrittenForSelection) {
-                const auto nFramesWrittenSoFar = (
-                    nFramesWrittenAtSelectionStart + nFramesWrittenForSelection
-                );
-                return 100 * nFramesWrittenSoFar / nFramesToWrite;
-            };
-        };
-
-        for (const auto& bounds : selection->getSelections()) {
-            completed = CSVStreamWriter::writeInChunks(
-                out,
-                *m_model,
-                bounds,
-                m_reporter,
-                m_delimiter,
-                m_options,
-                16384,
-                createProgressCalculator
-            );
-            if (!completed) {
-                break;
-            } 
-        }
-
         file.close();
         if (completed) {
             temp.moveToTarget();

--- a/data/fileio/CSVStreamWriter.h	Tue Apr 17 10:03:50 2018 +0100
+++ b/data/fileio/CSVStreamWriter.h	Tue Apr 17 10:03:50 2018 +0100
@@ -23,23 +23,70 @@
 #include "data/model/Model.h"
 #include <QString>
 #include <algorithm>
-
-namespace 
-{
-    const auto initProgressCalculator = [](sv_frame_t nFramesToWrite) {
-        return [nFramesToWrite](sv_frame_t nFramesWritten) {
-            return 100 * nFramesWritten / nFramesToWrite;
-        };
-    };
-} // namespace
+#include <numeric>
 
 namespace CSVStreamWriter
 {
 
-template <
-    class OutStream,
-    class ProgressCalculatorInit = decltype(initProgressCalculator)
->
+template <class OutStream>
+bool
+writeInChunks(OutStream& oss,
+              const Model& model,
+              const MultiSelection& regions,
+              ProgressReporter* reporter = nullptr,
+              QString delimiter = ",",
+              DataExportOptions options = DataExportDefaults,
+              const sv_frame_t blockSize = 16384)
+{
+    if (blockSize <= 0) return false;
+
+    // TODO, some form of checking validity of selections?
+    const auto nFramesToWrite = std::accumulate(
+        regions.getSelections().begin(),
+        regions.getSelections().end(),
+        0,
+        [](sv_frame_t acc, const Selection& current) -> sv_frame_t {
+            return acc + (current.getEndFrame() - current.getStartFrame());
+        }
+    );
+
+    const auto wasCancelled = [&reporter]() { 
+        return reporter && reporter->wasCancelled(); 
+    };
+
+    sv_frame_t nFramesWritten = 0;
+    int previousProgress = 0;
+
+    for (const auto& extents : regions.getSelections()) {
+        const auto startFrame = extents.getStartFrame();
+        const auto endFrame = extents.getEndFrame();
+        auto readPtr = startFrame;
+        while (readPtr < endFrame) {
+            if (wasCancelled()) return false;
+
+            const auto start = readPtr;
+            const auto end = std::min(start + blockSize, endFrame);
+
+            oss << model.toDelimitedDataStringSubsetWithOptions(
+                delimiter,
+                options,
+                start,
+                end
+            ) << (end < endFrame ? "\n" : "");
+            nFramesWritten += end - start;
+            const auto currentProgress = 100 * nFramesWritten / nFramesToWrite;
+            const bool hasIncreased = currentProgress > previousProgress;
+            if (hasIncreased) {
+                if (reporter) reporter->setProgress(currentProgress);
+                previousProgress = currentProgress;
+            }
+            readPtr = end;
+        }
+    }
+    return !wasCancelled(); // setProgress could process event loop
+}
+
+template <class OutStream>
 bool 
 writeInChunks(OutStream& oss,
               const Model& model,
@@ -47,48 +94,29 @@
               ProgressReporter* reporter = nullptr,
               QString delimiter = ",",
               DataExportOptions options = DataExportDefaults,
-              const sv_frame_t blockSize = 16384,
-              const ProgressCalculatorInit& initCalc = initProgressCalculator)
+              const sv_frame_t blockSize = 16384)
 {
-    if (blockSize <= 0) return false;
     const auto startFrame = extents.isEmpty() ?
         model.getStartFrame() : extents.getStartFrame();
     const auto endFrame = extents.isEmpty() ?
         model.getEndFrame() : extents.getEndFrame();
     const auto hasValidExtents = startFrame >= 0 && endFrame > startFrame;
     if (!hasValidExtents) return false;
-    const auto calculateProgress = initCalc(endFrame - startFrame);
-
-    auto readPtr = startFrame;
-    int previousPercentagePoint = 0;
-
-    const auto wasCancelled = [&reporter]() { 
-        return reporter && reporter->wasCancelled(); 
+    Selection all {
+        startFrame,
+        endFrame
     };
-
-    while (readPtr < endFrame) {
-        if (wasCancelled()) return false;
-
-        const auto start = readPtr;
-        const auto end = std::min(start + blockSize, endFrame);
-
-        oss << model.toDelimitedDataStringSubsetWithOptions(
-            delimiter,
-            options,
-            start,
-            end
-        ) << (end < endFrame ? "\n" : "");
-        const auto nFramesWritten = end - startFrame;
-        const auto currentPercentage = calculateProgress(nFramesWritten);
-        const bool hasIncreased = currentPercentage > previousPercentagePoint;
-
-        if (hasIncreased) {
-            if (reporter) reporter->setProgress(currentPercentage);
-            previousPercentagePoint = currentPercentage;
-        }
-        readPtr = end;
-    }
-    return !wasCancelled(); // setProgress could process event loop
+    MultiSelection regions;
+    regions.addSelection(all);
+    return CSVStreamWriter::writeInChunks(
+        oss,
+        model,
+        regions,
+        reporter,
+        delimiter,
+        options,
+        blockSize
+    );
 }
 
 template <class OutStream>