changeset 1452:6e9615bde1f9 streaming-csv-writer

For sparse models, write whole model to CSV in one go

--- a/base/Selection.h	Tue Apr 17 10:37:15 2018 +0100
+++ b/base/Selection.h	Tue Apr 17 10:37:50 2018 +0100
@@ -13,8 +13,8 @@
     COPYING included with this distribution for more information.
 */
 
-#ifndef _SELECTION_H_
-#define _SELECTION_H_
+#ifndef SV_SELECTION_H
+#define SV_SELECTION_H
 
 #include <cstddef>
 #include <set>

--- a/data/fileio/CSVFileWriter.cpp	Tue Apr 17 10:37:15 2018 +0100
+++ b/data/fileio/CSVFileWriter.cpp	Tue Apr 17 10:37:50 2018 +0100
@@ -67,11 +67,11 @@
     };
     MultiSelection selections;
     selections.addSelection(all);
-    writeSelection(&selections); 
+    writeSelection(selections); 
 }
 
 void
-CSVFileWriter::writeSelection(MultiSelection *selection)
+CSVFileWriter::writeSelection(MultiSelection selection)
 {
     try {
         TempWriteFile temp(m_path);
@@ -85,13 +85,24 @@
     
         QTextStream out(&file);
 
+        sv_frame_t blockSize = 65536;
+
+        if (m_model->isSparse()) {
+            // Write the whole in one go, as re-seeking for each block
+            // may be very costly otherwise
+            sv_frame_t startFrame, endFrame;
+            selection.getExtents(startFrame, endFrame);
+            blockSize = endFrame - startFrame;
+        }
+        
         bool completed = CSVStreamWriter::writeInChunks(
             out,
             *m_model,
-            *selection,
+            selection,
             m_reporter,
             m_delimiter,
-            m_options
+            m_options,
+            blockSize
         );
 
         file.close();

--- a/data/fileio/CSVFileWriter.h	Tue Apr 17 10:37:15 2018 +0100
+++ b/data/fileio/CSVFileWriter.h	Tue Apr 17 10:37:50 2018 +0100
@@ -13,8 +13,8 @@
     COPYING included with this distribution for more information.
 */
 
-#ifndef _CSV_FILE_WRITER_H_
-#define _CSV_FILE_WRITER_H_
+#ifndef SV_CSV_FILE_WRITER_H
+#define SV_CSV_FILE_WRITER_H
 
 #include <QObject>
 #include <QString>
@@ -50,7 +50,7 @@
     virtual QString getError() const;
 
     virtual void write();
-    virtual void writeSelection(MultiSelection *selection);
+    virtual void writeSelection(MultiSelection selection);
 
 protected:
     QString m_path;

--- a/data/fileio/CSVStreamWriter.h	Tue Apr 17 10:37:15 2018 +0100
+++ b/data/fileio/CSVStreamWriter.h	Tue Apr 17 10:37:50 2018 +0100
@@ -13,8 +13,8 @@
     COPYING included with this distribution for more information.
 */
 
-#ifndef CSV_STREAM_WRITER_H
-#define CSV_STREAM_WRITER_H
+#ifndef SV_CSV_STREAM_WRITER_H
+#define SV_CSV_STREAM_WRITER_H
 
 #include "base/BaseTypes.h"
 #include "base/Selection.h"
@@ -146,4 +146,4 @@
     );
 }
 } // namespace CSVStreamWriter
-#endif
\ No newline at end of file
+#endif

--- a/data/model/Model.h	Tue Apr 17 10:37:15 2018 +0100
+++ b/data/model/Model.h	Tue Apr 17 10:37:50 2018 +0100
@@ -95,6 +95,11 @@
     virtual QString getTypeName() const = 0;
 
     /**
+     * Return true if this is a sparse model.
+     */
+    virtual bool isSparse() const { return false; }
+    
+    /**
      * Mark the model as abandoning. This means that the application
      * no longer needs it, so it can stop doing any background
      * calculations it may be involved in. Note that as far as the

--- a/data/model/SparseModel.h	Tue Apr 17 10:37:15 2018 +0100
+++ b/data/model/SparseModel.h	Tue Apr 17 10:37:50 2018 +0100
@@ -13,8 +13,8 @@
     COPYING included with this distribution for more information.
 */
 
-#ifndef _SPARSE_MODEL_H_
-#define _SPARSE_MODEL_H_
+#ifndef SV_SPARSE_MODEL_H
+#define SV_SPARSE_MODEL_H
 
 #include "Model.h"
 #include "TabularModel.h"
@@ -145,6 +145,8 @@
 
     virtual bool hasTextLabels() const { return m_hasTextLabels; }
 
+    virtual bool isSparse() const { return true; }
+
     QString getTypeName() const { return tr("Sparse"); }
 
     virtual QString getXmlOutputType() const { return "sparse"; }