annotate data/fileio/CSVStreamWriter.h @ 1452:6e9615bde1f9 streaming-csv-writer

For sparse models, write whole model to CSV in one go
author Chris Cannam <cannam@all-day-breakfast.com>
date Tue, 17 Apr 2018 10:37:50 +0100
parents deabf9fd3d28
children 743c38b209d0
rev   line source
dev@1430 1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
dev@1430 2
dev@1430 3 /*
dev@1430 4 Sonic Visualiser
dev@1430 5 An audio file viewer and annotation editor.
dev@1430 6 Centre for Digital Music, Queen Mary, University of London.
dev@1438 7 This file copyright 2017 Queen Mary, University of London.
dev@1430 8
dev@1430 9 This program is free software; you can redistribute it and/or
dev@1430 10 modify it under the terms of the GNU General Public License as
dev@1430 11 published by the Free Software Foundation; either version 2 of the
dev@1430 12 License, or (at your option) any later version. See the file
dev@1430 13 COPYING included with this distribution for more information.
dev@1430 14 */
dev@1430 15
cannam@1452 16 #ifndef SV_CSV_STREAM_WRITER_H
cannam@1452 17 #define SV_CSV_STREAM_WRITER_H
dev@1430 18
dev@1434 19 #include "base/BaseTypes.h"
dev@1434 20 #include "base/Selection.h"
dev@1434 21 #include "base/ProgressReporter.h"
dev@1434 22 #include "base/DataExportOptions.h"
dev@1434 23 #include "data/model/Model.h"
dev@1434 24 #include <QString>
dev@1434 25 #include <algorithm>
dev@1445 26 #include <numeric>
dev@1441 27
dev@1438 28 namespace CSVStreamWriter
dev@1430 29 {
dev@1430 30
dev@1445 31 template <class OutStream>
dev@1445 32 bool
dev@1445 33 writeInChunks(OutStream& oss,
dev@1445 34 const Model& model,
dev@1445 35 const MultiSelection& regions,
dev@1445 36 ProgressReporter* reporter = nullptr,
dev@1445 37 QString delimiter = ",",
dev@1445 38 DataExportOptions options = DataExportDefaults,
dev@1445 39 const sv_frame_t blockSize = 16384)
dev@1445 40 {
dev@1447 41 const auto selections = regions.getSelections();
dev@1447 42 if (blockSize <= 0 || selections.empty()) return false;
dev@1445 43
dev@1445 44 // TODO, some form of checking validity of selections?
dev@1445 45 const auto nFramesToWrite = std::accumulate(
dev@1447 46 selections.begin(),
dev@1447 47 selections.end(),
dev@1445 48 0,
dev@1445 49 [](sv_frame_t acc, const Selection& current) -> sv_frame_t {
dev@1445 50 return acc + (current.getEndFrame() - current.getStartFrame());
dev@1445 51 }
dev@1445 52 );
dev@1447 53 const auto finalFrameOfLastRegion = (*selections.crbegin()).getEndFrame();
dev@1445 54
dev@1445 55 const auto wasCancelled = [&reporter]() {
dev@1445 56 return reporter && reporter->wasCancelled();
dev@1445 57 };
dev@1445 58
dev@1445 59 sv_frame_t nFramesWritten = 0;
dev@1445 60 int previousProgress = 0;
dev@1445 61
dev@1447 62 for (const auto& extents : selections) {
dev@1445 63 const auto startFrame = extents.getStartFrame();
dev@1445 64 const auto endFrame = extents.getEndFrame();
dev@1445 65 auto readPtr = startFrame;
dev@1445 66 while (readPtr < endFrame) {
dev@1445 67 if (wasCancelled()) return false;
dev@1445 68
dev@1445 69 const auto start = readPtr;
dev@1445 70 const auto end = std::min(start + blockSize, endFrame);
dev@1449 71 const auto data = model.toDelimitedDataStringSubsetWithOptions(
dev@1445 72 delimiter,
dev@1445 73 options,
dev@1445 74 start,
dev@1445 75 end
dev@1449 76 ).trimmed();
dev@1449 77
dev@1449 78 if ( data != "" ) {
dev@1449 79 oss << data << (end < finalFrameOfLastRegion ? "\n" : "");
dev@1449 80 }
dev@1449 81
dev@1445 82 nFramesWritten += end - start;
dev@1445 83 const auto currentProgress = 100 * nFramesWritten / nFramesToWrite;
dev@1445 84 const bool hasIncreased = currentProgress > previousProgress;
dev@1445 85 if (hasIncreased) {
dev@1445 86 if (reporter) reporter->setProgress(currentProgress);
dev@1445 87 previousProgress = currentProgress;
dev@1445 88 }
dev@1445 89 readPtr = end;
dev@1445 90 }
dev@1445 91 }
dev@1445 92 return !wasCancelled(); // setProgress could process event loop
dev@1445 93 }
dev@1445 94
dev@1445 95 template <class OutStream>
dev@1441 96 bool
dev@1441 97 writeInChunks(OutStream& oss,
dev@1441 98 const Model& model,
dev@1441 99 const Selection& extents,
dev@1441 100 ProgressReporter* reporter = nullptr,
dev@1441 101 QString delimiter = ",",
dev@1441 102 DataExportOptions options = DataExportDefaults,
dev@1445 103 const sv_frame_t blockSize = 16384)
dev@1434 104 {
dev@1441 105 const auto startFrame = extents.isEmpty() ?
dev@1434 106 model.getStartFrame() : extents.getStartFrame();
dev@1441 107 const auto endFrame = extents.isEmpty() ?
dev@1434 108 model.getEndFrame() : extents.getEndFrame();
dev@1441 109 const auto hasValidExtents = startFrame >= 0 && endFrame > startFrame;
dev@1441 110 if (!hasValidExtents) return false;
dev@1445 111 Selection all {
dev@1445 112 startFrame,
dev@1445 113 endFrame
dev@1434 114 };
dev@1445 115 MultiSelection regions;
dev@1445 116 regions.addSelection(all);
dev@1445 117 return CSVStreamWriter::writeInChunks(
dev@1445 118 oss,
dev@1445 119 model,
dev@1445 120 regions,
dev@1445 121 reporter,
dev@1445 122 delimiter,
dev@1445 123 options,
dev@1445 124 blockSize
dev@1445 125 );
dev@1434 126 }
dev@1434 127
dev@1434 128 template <class OutStream>
dev@1441 129 bool
dev@1441 130 writeInChunks(OutStream& oss,
dev@1441 131 const Model& model,
dev@1441 132 ProgressReporter* reporter = nullptr,
dev@1441 133 QString delimiter = ",",
dev@1441 134 DataExportOptions options = DataExportDefaults,
dev@1441 135 const sv_frame_t blockSize = 16384)
dev@1434 136 {
dev@1434 137 const Selection empty;
dev@1444 138 return CSVStreamWriter::writeInChunks(
dev@1434 139 oss,
dev@1434 140 model,
dev@1434 141 empty,
dev@1434 142 reporter,
dev@1434 143 delimiter,
dev@1434 144 options,
dev@1434 145 blockSize
dev@1434 146 );
dev@1434 147 }
dev@1438 148 } // namespace CSVStreamWriter
cannam@1452 149 #endif