Mercurial > hg > svcore
changeset 1434:0684c6698e3f streaming-csv-writer
Added utility function for splitting a model selection into chunks and writing to a stream.
author | Lucas Thompson <dev@lucas.im> |
---|---|
date | Tue, 17 Apr 2018 10:03:49 +0100 |
parents | c0f69bddea12 |
children | 365876627fb0 |
files | data/fileio/CSVStreamWriter.cpp data/fileio/CSVStreamWriter.h data/fileio/test/CSVStreamWriterTest.h data/fileio/test/files.pri files.pri |
diffstat | 4 files changed, 222 insertions(+), 7 deletions(-) [+] |
line wrap: on
line diff
--- a/data/fileio/CSVStreamWriter.h Tue Apr 17 10:03:49 2018 +0100 +++ b/data/fileio/CSVStreamWriter.h Tue Apr 17 10:03:49 2018 +0100 @@ -16,11 +16,85 @@ #ifndef _CSV_STREAM_WRITER_H_ #define _CSV_STREAM_WRITER_H_ -class CSVStreamWriter +#include "base/BaseTypes.h" +#include "base/Selection.h" +#include "base/ProgressReporter.h" +#include "base/DataExportOptions.h" +#include "data/model/Model.h" +#include <QString> +#include <algorithm> + +namespace CSV { +using Completed = bool; -}; +template <class OutStream> +auto writeToStreamInChunks( + OutStream& oss, + const Model& model, + const Selection& extents, + ProgressReporter* reporter = nullptr, + QString delimiter = ",", + DataExportOptions options = DataExportDefaults, + const sv_frame_t blockSize = 16384 +) -> Completed +{ + if (blockSize <= 0) return false; + sv_frame_t readPtr = extents.isEmpty() ? + model.getStartFrame() : extents.getStartFrame(); + sv_frame_t endFrame = extents.isEmpty() ? + model.getEndFrame() : extents.getEndFrame(); + int previousPercentagePoint = 0; -#endif + const auto wasCancelled = [&reporter]() { + return reporter && reporter->wasCancelled(); + }; - + while (readPtr < endFrame) { + if (wasCancelled()) return false; + + const auto start = readPtr; + const auto end = std::min(start + blockSize, endFrame); + + oss << model.toDelimitedDataStringSubsetWithOptions( + delimiter, + options, + start, + end + ) << (end < endFrame ? "\n" : ""); + + const auto currentPercentage = 100 * end / endFrame; + const bool hasIncreased = currentPercentage > previousPercentagePoint; + + if (hasIncreased) { + if (reporter) reporter->setProgress(currentPercentage); + previousPercentagePoint = currentPercentage; + } + readPtr = end; + } + return !wasCancelled(); // setProgress could process event loop +} + +template <class OutStream> +auto writeToStreamInChunks( + OutStream& oss, + const Model& model, + ProgressReporter* reporter = nullptr, + QString delimiter = ",", + DataExportOptions options = DataExportDefaults, + const sv_frame_t blockSize = 16384 +) -> Completed +{ + const Selection empty; + return CSV::writeToStreamInChunks( + oss, + model, + empty, + reporter, + delimiter, + options, + blockSize + ); +} +} // namespace +#endif \ No newline at end of file
--- a/data/fileio/test/CSVStreamWriterTest.h Tue Apr 17 10:03:49 2018 +0100 +++ b/data/fileio/test/CSVStreamWriterTest.h Tue Apr 17 10:03:49 2018 +0100 @@ -4,7 +4,7 @@ Sonic Visualiser An audio file viewer and annotation editor. Centre for Digital Music, Queen Mary, University of London. - This file copyright 2013 Chris Cannam. + This file copyright 2017 Lucas Thompson. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -16,18 +16,158 @@ #ifndef TEST_CSV_STREAM_H #define TEST_CSV_STREAM_H +#include <QtTest> #include <QObject> +#include <sstream> +#include <functional> + +#include "base/ProgressReporter.h" +#include "base/DataExportOptions.h" #include "../CSVStreamWriter.h" +#include "../../model/test/MockWaveModel.h" + +namespace +{ + class StubReporter : public ProgressReporter + { + public: + StubReporter( std::function<bool()> isCancelled ) + : m_isCancelled(isCancelled) {} + bool isDefinite() const override { return true; } + void setDefinite(bool) override {} + bool wasCancelled() const override { return m_isCancelled(); } + void setMessage(QString) override {} + void setProgress(int) override { ++m_calls; } + size_t getCallCount() const { return m_calls; } + void reset() { m_calls = 0; } + private: + size_t m_calls = 0; + std::function<bool()> m_isCancelled; + }; +} // namespace class CSVStreamWriterTest : public QObject { Q_OBJECT public: + std::string getExpectedString() + { + return + { + "0,0,0\n" + "1,0,0\n" + "2,0,0\n" + "3,0,0\n" + "4,1,1\n" + "5,1,1\n" + "6,1,1\n" + "7,1,1\n" + "8,1,1\n" + "9,1,1\n" + "10,1,1\n" + "11,1,1\n" + "12,1,1\n" + "13,1,1\n" + "14,1,1\n" + "15,1,1\n" + "16,1,1\n" + "17,1,1\n" + "18,1,1\n" + "19,1,1\n" + "20,0,0\n" + "21,0,0\n" + "22,0,0\n" + "23,0,0" + }; + } private slots: - void toDo() + void simpleValidOutput() { + MockWaveModel mwm({ DC, DC }, 16, 4); + std::ostringstream oss; + const auto result = CSV::writeToStreamInChunks(oss, mwm); + QVERIFY( oss.str() == getExpectedString() ); + QVERIFY( result ); + } + + void callsReporterCorrectTimes() + { + MockWaveModel mwm({ DC, DC }, 16, 4); + StubReporter reporter { []() -> bool { return false; } }; + const auto expected = getExpectedString(); + + std::ostringstream oss; + const auto writeStreamWithBlockSize = [&](int blockSize) { + return CSV::writeToStreamInChunks( + oss, + mwm, + &reporter, + ",", + DataExportDefaults, + blockSize + ); + }; + + const auto reset = [&]() { + oss.str({}); + reporter.reset(); + }; + + const auto nonIntegerMultipleResult = writeStreamWithBlockSize(5); + QVERIFY( nonIntegerMultipleResult ); + QVERIFY( reporter.getCallCount() == 5 /* 4.8 rounded up */ ); + QVERIFY( oss.str() == expected ); + reset(); + + const auto integerMultiple = writeStreamWithBlockSize(2); + QVERIFY( integerMultiple ); + QVERIFY( reporter.getCallCount() == 12 ); + QVERIFY( oss.str() == expected ); + reset(); + + const auto largerThanNumberOfSamples = writeStreamWithBlockSize(100); + QVERIFY( largerThanNumberOfSamples ); + QVERIFY( reporter.getCallCount() == 1 ); + QVERIFY( oss.str() == expected ); + reset(); + + const auto zero = writeStreamWithBlockSize(0); + QVERIFY( zero == false ); + QVERIFY( reporter.getCallCount() == 0 ); + } + + void isCancellable() + { + MockWaveModel mwm({ DC, DC }, 16, 4); + StubReporter reporter { []() -> bool { return true; } }; + + std::ostringstream oss; + const auto cancelImmediately = CSV::writeToStreamInChunks( + oss, + mwm, + &reporter, + ",", + DataExportDefaults, + 4 + ); + QVERIFY( cancelImmediately == false ); + QVERIFY( reporter.getCallCount() == 0 ); + + StubReporter cancelMidway { + [&]() { return cancelMidway.getCallCount() == 3; } + }; + const auto cancelledMidway = CSV::writeToStreamInChunks( + oss, + mwm, + &cancelMidway, + ",", + DataExportDefaults, + 4 + ); + QVERIFY( cancelMidway.getCallCount() == 3 ); + QVERIFY( cancelledMidway == false ); } };
--- a/data/fileio/test/files.pri Tue Apr 17 10:03:49 2018 +0100 +++ b/data/fileio/test/files.pri Tue Apr 17 10:03:49 2018 +0100 @@ -1,6 +1,7 @@ TEST_HEADERS += \ ../../../test/TestHelper.h \ + ../../model/test/MockWaveModel.h \ AudioFileReaderTest.h \ AudioFileWriterTest.h \ AudioTestData.h \ @@ -9,4 +10,5 @@ CSVStreamWriterTest.h TEST_SOURCES += \ + ../../model/test/MockWaveModel.cpp \ svcore-data-fileio-test.cpp
--- a/files.pri Tue Apr 17 10:03:49 2018 +0100 +++ b/files.pri Tue Apr 17 10:03:49 2018 +0100 @@ -181,7 +181,6 @@ data/fileio/CSVFileReader.cpp \ data/fileio/CSVFileWriter.cpp \ data/fileio/CSVFormat.cpp \ - data/fileio/CSVStreamWriter.cpp \ data/fileio/DataFileReaderFactory.cpp \ data/fileio/FileReadThread.cpp \ data/fileio/FileSource.cpp \